Support region callbacks when no channel is specified

Regression introduced with 0ff8eaf

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,6 @@
 Describe your proposal.
 
-Mention any issue this PR is resolves or is related to.
+Mention any issue this PR resolves or is related to.
 
 ---
 

CMakeLists.txt

@@ -6,7 +6,7 @@
 #                  copied verbatim in the file "LICENSE"                       #
 ################################################################################
 
-cmake_minimum_required(VERSION 3.9.4 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.10 FATAL_ERROR)
 
 
 # Project ######################################################################
@@ -19,6 +19,12 @@ get_git_version()
 project(FairMQ VERSION ${PROJECT_VERSION} LANGUAGES CXX)
 message(STATUS "${BWhite}${PROJECT_NAME}${CR} ${PROJECT_GIT_VERSION} from ${PROJECT_DATE}")
 
+if(BUILD_OFI_TRANSPORT)
+  set(PROJECT_MIN_CXX_STANDARD 14)
+else()
+  set(PROJECT_MIN_CXX_STANDARD 11)
+endif()
+
 set_fairmq_defaults()
 
 include(CTest)
@@ -32,6 +38,7 @@ cmake_dependent_option(BUILD_TESTING "Build tests." OFF "BUILD_FAIRMQ" OFF)
 cmake_dependent_option(BUILD_NANOMSG_TRANSPORT "Build nanomsg transport." OFF "BUILD_FAIRMQ" OFF)
 cmake_dependent_option(BUILD_OFI_TRANSPORT "Build experimental OFI transport." OFF "BUILD_FAIRMQ" OFF)
 cmake_dependent_option(BUILD_DDS_PLUGIN "Build DDS plugin." OFF "BUILD_FAIRMQ" OFF)
+cmake_dependent_option(BUILD_PMIX_PLUGIN "Build PMIx plugin." OFF "BUILD_FAIRMQ" OFF)
 cmake_dependent_option(BUILD_EXAMPLES "Build FairMQ examples." ON "BUILD_FAIRMQ" OFF)
 option(BUILD_DOCS "Build FairMQ documentation." OFF)
 option(FAST_BUILD "Fast production build. Not recommended for development." OFF)
@@ -47,35 +54,69 @@ set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
 set(THREADS_PREFER_PTHREAD_FLAG TRUE)
 find_package(Threads REQUIRED)
 
-if(BUILD_FAIRMQ)
-  find_package2(PUBLIC Boost VERSION 1.64 REQUIRED
-    COMPONENTS container program_options thread system filesystem regex date_time signals
-  )
-  find_package2(PUBLIC FairLogger VERSION 1.2.0 REQUIRED)
-  find_package2(PRIVATE ZeroMQ VERSION 4.1.5 REQUIRED)
-endif()
-
 if(BUILD_NANOMSG_TRANSPORT)
   find_package2(PRIVATE nanomsg REQUIRED)
   set(PROJECT_nanomsg_VERSION 1.1.3) # Once upstream releases 1.1.5, we should bump again and use version check
-  find_package2(PRIVATE msgpack VERSION 3.1.0 REQUIRED)
 endif()
 
 if(BUILD_OFI_TRANSPORT)
-  find_package2(PRIVATE OFI VERSION 1.6.0 REQUIRED COMPONENTS fi_sockets fi_verbs)
-  find_package2(PRIVATE Protobuf VERSION 3.4.0 REQUIRED)
+  find_package2(PRIVATE asiofi REQUIRED
+    VERSION 0.2.0
+  )
+  find_package2(PRIVATE OFI REQUIRED
+    VERSION ${asiofi_OFI_VERSION}
+    COMPONENTS ${asiofi_OFI_COMPONENTS}
+  )
+  find_package2(PRIVATE AZMQ REQUIRED)
+  set(PROJECT_AZMQ_VERSION 1.0.2)
+endif()
+
+if(BUILD_NANOMSG_TRANSPORT)
+  find_package2(PRIVATE msgpack REQUIRED
+    VERSION 3.1.0
+  )
+endif()
+
+if(BUILD_FAIRMQ)
+  find_package2(PUBLIC Boost REQUIRED
+    VERSION 1.64 ${asiofi_Boost_VERSION}
+
+    COMPONENTS
+    container
+    program_options
+    filesystem
+    date_time
+    regex
+    ${asiofi_Boost_COMPONENTS}
+  )
+  find_package2(PUBLIC FairLogger REQUIRED
+    VERSION 1.2.0
+  )
+  find_package2(PRIVATE ZeroMQ REQUIRED
+    VERSION 4.1.5
+  )
 endif()
 
 if(BUILD_DDS_PLUGIN)
   find_package2(PRIVATE DDS VERSION 2.2 REQUIRED)
 endif()
 
+if(BUILD_PMIX_PLUGIN)
+  find_package2(PRIVATE PMIx VERSION 2.1.4 REQUIRED)
+endif()
+
 if(BUILD_TESTING)
-  find_package2(PRIVATE GTest VERSION 1.7.0 REQUIRED)
+  find_package2(PRIVATE GTest REQUIRED
+    VERSION 1.7.0
+  )
 endif()
 
 if(BUILD_DOCS)
-  find_package2(PRIVATE Doxygen VERSION 1.8.8 REQUIRED COMPONENTS dot OPTIONAL_COMPONENTS mscgen dia)
+  find_package2(PRIVATE Doxygen REQUIRED
+    VERSION 1.8.8
+    COMPONENTS dot
+    OPTIONAL_COMPONENTS mscgen dia
+  )
 endif()
 ################################################################################
 
@@ -101,7 +142,7 @@ endif()
 if(BUILD_DOCS)
   set(DOXYGEN_OUTPUT_DIRECTORY doxygen)
   set(DOXYGEN_PROJECT_NUMBER ${PROJECT_GIT_VERSION})
-  set(DOXYGEN_PROJECT_BRIEF "C++ Message Passing Framework")
+  set(DOXYGEN_PROJECT_BRIEF "C++ Message Queuing Library and Framework")
   set(DOXYGEN_USE_MDFILE_AS_MAINPAGE README.md)
   set(DOXYGEN_HTML_FOOTER docs/footer.html)
   doxygen_add_docs(doxygen README.md fairmq)
@@ -120,6 +161,9 @@ endif()
 if(BUILD_DDS_PLUGIN)
   list(APPEND PROJECT_PACKAGE_COMPONENTS dds_plugin)
 endif()
+if(BUILD_PMIX_PLUGIN)
+  list(APPEND PROJECT_PACKAGE_COMPONENTS pmix_plugin)
+endif()
 if(BUILD_NANOMSG_TRANSPORT)
   list(APPEND PROJECT_PACKAGE_COMPONENTS nanomsg_transport)
 endif()
@@ -151,7 +195,7 @@ if(BUILD_DDS_PLUGIN)
   )
 endif()
 if(BUILD_OFI_TRANSPORT)
-  install(FILES cmake/FindOFI.cmake
+  install(FILES cmake/FindAZMQ.cmake
     DESTINATION ${PROJECT_INSTALL_CMAKEMODDIR}
   )
 endif()
@@ -166,6 +210,8 @@ install_cmake_package()
 
 
 # Summary ######################################################################
+message(STATUS "  ")
+message(STATUS "  ${Cyan}CXX STANDARD${CR}       ${BGreen}C++${CMAKE_CXX_STANDARD}${CR} (>= C++${PROJECT_MIN_CXX_STANDARD}, change with ${BMagenta}-DCMAKE_CXX_STANDARD=17${CR})")
 if(CMAKE_CXX_FLAGS)
   message(STATUS "  ")
   message(STATUS "  ${Cyan}GLOBAL CXX FLAGS${CR}  ${BGreen}${CMAKE_CXX_FLAGS}${CR}")
@@ -218,6 +264,8 @@ if(PROJECT_PACKAGE_DEPENDENCIES)
     elseif(${dep} STREQUAL msgpack)
       get_target_property(msgpack_include msgpackc-cxx INTERFACE_INCLUDE_DIRECTORIES)
       get_filename_component(prefix ${msgpack_include}/.. ABSOLUTE)
+    elseif(${dep} STREQUAL asiofi)
+      set(prefix ${asiofi_ROOT})
     elseif(${dep} STREQUAL OFI)
       get_filename_component(prefix ${${dep}_INCLUDE_DIRS}/.. ABSOLUTE)
     elseif(${dep} STREQUAL nanomsg)
@@ -257,9 +305,9 @@ else()
 endif()
 message(STATUS "  ${BWhite}nanomsg_transport${CR}  ${nn_summary}")
 if(BUILD_OFI_TRANSPORT)
-  set(ofi_summary "${BGreen}YES${CR}    EXPERIMENTAL (disable with ${BMagenta}-DBUILD_OFI_TRANSPORT=OFF${CR})")
+  set(ofi_summary "${BGreen}YES${CR}    EXPERIMENTAL (requires C++14) (disable with ${BMagenta}-DBUILD_OFI_TRANSPORT=OFF${CR})")
 else()
-  set(ofi_summary "${BRed} NO${CR}    EXPERIMENTAL (default, enable with ${BMagenta}-DBUILD_OFI_TRANSPORT=ON${CR})")
+  set(ofi_summary "${BRed} NO${CR}    EXPERIMENTAL (requires C++14) (default, enable with ${BMagenta}-DBUILD_OFI_TRANSPORT=ON${CR})")
 endif()
 message(STATUS "  ${BWhite}ofi_transport${CR}      ${ofi_summary}")
 if(BUILD_DDS_PLUGIN)
@@ -268,6 +316,12 @@ else()
   set(dds_summary "${BRed} NO${CR}    (default, enable with ${BMagenta}-DBUILD_DDS_PLUGIN=ON${CR})")
 endif()
 message(STATUS "  ${BWhite}dds_plugin${CR}         ${dds_summary}")
+if(BUILD_PMIX_PLUGIN)
+  set(pmix_summary "${BGreen}YES${CR}    (disable with ${BMagenta}-DBUILD_PMIX_PLUGIN=OFF${CR})")
+else()
+  set(pmix_summary "${BRed} NO${CR}    (default, enable with ${BMagenta}-DBUILD_PMIX_PLUGIN=ON${CR})")
+endif()
+message(STATUS "  ${BWhite}pmix_plugin${CR}        ${pmix_summary}")
 if(BUILD_EXAMPLES)
   set(examples_summary "${BGreen}YES${CR}    (default, disable with ${BMagenta}-DBUILD_EXAMPLES=OFF${CR})")
 else()

FairMQTest.cmake

@@ -27,6 +27,7 @@ Set(configure_options "${configure_options};-DCTEST_USE_LAUNCHERS=${CTEST_USE_LA
 Set(configure_options "${configure_options};-DDISABLE_COLOR=ON")
 Set(configure_options "${configure_options};-DCMAKE_PREFIX_PATH=$ENV{SIMPATH}")
 Set(configure_options "${configure_options};-DBUILD_NANOMSG_TRANSPORT=ON")
+# Set(configure_options "${configure_options};-DBUILD_OFI_TRANSPORT=ON")
 Set(configure_options "${configure_options};-DBUILD_DDS_PLUGIN=ON")
 Set(configure_options "${configure_options};-DFAST_BUILD=ON")
 Set(configure_options "${configure_options};-DCOTIRE_MAXIMUM_NUMBER_OF_UNITY_INCLUDES=-j$ENV{number_of_processors}")
@@ -57,7 +58,8 @@ Ctest_Configure(BUILD "${CTEST_BINARY_DIRECTORY}"
 Ctest_Build(BUILD "${CTEST_BINARY_DIRECTORY}")
 
 Ctest_Test(BUILD "${CTEST_BINARY_DIRECTORY}" 
-           PARALLEL_LEVEL $ENV{number_of_processors}
+           # PARALLEL_LEVEL $ENV{number_of_processors}
+           PARALLEL_LEVEL 1
            RETURN_VALUE _ctest_test_ret_val
           )
 If("$ENV{do_codecov_upload}")

Jenkinsfile

@@ -66,14 +66,15 @@ pipeline{
       steps{
         script {
           def build_jobs = jobMatrix('alfa-ci/build', [
-            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',          fairsoft: 'fairmq_dev'],
+            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',        fairsoft: 'fairmq_dev'],
             [os: 'MacOS10.13', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
+            [os: 'MacOS10.14', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
           ]) { spec, label ->
             sh './Dart.sh alfa_ci Dart.cfg'
           }
 
           def profile_jobs = jobMatrix('alfa-ci/codecov', [
-            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',          fairsoft: 'fairmq_dev'],
+            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',        fairsoft: 'fairmq_dev'],
           ]) { spec, label ->
             withCredentials([string(credentialsId: 'fairmq_codecov_token', variable: 'CODECOV_TOKEN')]) {
               sh './Dart.sh codecov Dart.cfg'

Jenkinsfile.nightly

@@ -63,8 +63,9 @@ pipeline{
       steps{
         script {
           parallel(buildMatrix([
-            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',         fairsoft: 'fairmq_dev'],
-            [os: 'MacOS10.13', arch: 'x86_64', compiler: 'AppleLLVM10.0.0',  fairsoft: 'fairmq_dev'],
+            [os: 'Debian8',    arch: 'x86_64', compiler: 'gcc8.1.0',        fairsoft: 'fairmq_dev'],
+            [os: 'MacOS10.13', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
+            [os: 'MacOS10.14', arch: 'x86_64', compiler: 'AppleLLVM10.0.0', fairsoft: 'fairmq_dev'],
           ]) { spec, label ->
             sh './Dart.sh Nightly Dart.cfg'
             sh './Dart.sh Profile Dart.cfg'

README.md

@@ -38,7 +38,7 @@ a simulation, reconstruction and analysis framework.
   * PUBLIC: [**Boost**](https://www.boost.org/), [**FairLogger**](https://github.com/FairRootGroup/FairLogger)
   * BUILD: [CMake](https://cmake.org/), [GTest](https://github.com/google/googletest), [Doxygen](http://www.doxygen.org/)
   * PRIVATE: [ZeroMQ](http://zeromq.org/), [Msgpack](https://msgpack.org/index.html), [nanomsg](http://nanomsg.org/),
-[OFI](https://ofiwg.github.io/libfabric/), [Protobuf](https://developers.google.com/protocol-buffers/), [DDS](http://dds.gsi.de)
+[asiofi](https://github.com/FairRootGroup/asiofi), [DDS](http://dds.gsi.de), [PMIx](https://pmix.org/), [AZMQ](https://github.com/zeromq/azmq), [asiofi](https://github.com/FairRootGroup/asiofi)
 
   Supported platforms: Linux and MacOS.
 
@@ -51,7 +51,7 @@ cmake -DCMAKE_INSTALL_PREFIX=./fairmq_install ../fairmq
 cmake --build . --target install
 ```
 
-If dependencies are not installed in standard system directories, you can hint the installation location via `-DCMAKE_PREFIX_PATH=...` or per dependency via `-D{DEPENDENCY}_ROOT=...`. `{DEPENDENCY}` can be `GTEST`, `BOOST`, `FAIRLOGGER`, `ZEROMQ`, `MSGPACK`, `NANOMSG`, `OFI`, `PROTOBUF`, or `DDS` (`*_ROOT` variables can also be environment variables).
+If dependencies are not installed in standard system directories, you can hint the installation location via `-DCMAKE_PREFIX_PATH=...` or per dependency via `-D{DEPENDENCY}_ROOT=...`. `{DEPENDENCY}` can be `GTEST`, `BOOST`, `FAIRLOGGER`, `ZEROMQ`, `MSGPACK`, `NANOMSG`, `OFI`, `PMIX`, `ASIOFI`, `AZMQ` or `DDS` (`*_ROOT` variables can also be environment variables).
 
 ## Usage
 
@@ -104,6 +104,7 @@ On command line:
   * `-DBUILD_NANOMSG_TRANSPORT=ON` enables building of nanomsg transport.
   * `-DBUILD_OFI_TRANSPORT=ON` enables building of the experimental OFI transport.
   * `-DBUILD_DDS_PLUGIN=ON` enables building of the DDS plugin.
+  * `-DBUILD_PMIX_PLUGIN=ON` enables building of the PMIx plugin.
   * `-DBUILD_DOCS=ON` enables building of API docs.
   * You can hint non-system installations for dependent packages, see the #Installation section above
 
@@ -119,7 +120,7 @@ After the `find_package(FairMQ)` call the following CMake variables are defined:
 | `${FairMQ_#COMPONENT#_FOUND}` | `TRUE` if this component was built |
 | `${FairMQ_VERSION}` | the version in format `MAJOR.MINOR.PATCH` |
 | `${FairMQ_GIT_VERSION}` | the version in the format returned by `git describe --tags --dirty --match "v*"` |
-| `${FairMQ_ROOT}` | the actual installation prefix, notice the difference to the hint variable `FAIRMQ_ROOT` |
+| `${FairMQ_PREFIX}` | the actual installation prefix |
 | `${FairMQ_BINDIR}` | the installation bin directory |
 | `${FairMQ_INCDIR}` | the installation include directory |
 | `${FairMQ_LIBDIR}` | the installation lib directory |

cmake/FairMQConfig.cmake.in

@@ -13,7 +13,7 @@ set(@PROJECT_NAME@_VERSION @PROJECT_VERSION@)
 set(@PROJECT_NAME@_GIT_VERSION @PROJECT_GIT_VERSION@)
 set(@PROJECT_NAME@_GIT_DATE @PROJECT_GIT_DATE@)
 
-set_and_check(@PROJECT_NAME@_ROOT @PACKAGE_CMAKE_INSTALL_PREFIX@)
+set_and_check(@PROJECT_NAME@_PREFIX @PACKAGE_CMAKE_INSTALL_PREFIX@)
 set_and_check(@PROJECT_NAME@_BINDIR @PACKAGE_CMAKE_INSTALL_PREFIX@/@PROJECT_INSTALL_BINDIR@)
 set_and_check(@PROJECT_NAME@_INCDIR @PACKAGE_CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@)
 set_and_check(@PROJECT_NAME@_LIBDIR @PACKAGE_CMAKE_INSTALL_PREFIX@/@PROJECT_INSTALL_LIBDIR@)

cmake/FairMQLib.cmake

@@ -40,6 +40,7 @@ macro(set_fairmq_cmake_policies)
     CMP0042 # MACOSX_RPATH is enabled by default.
     CMP0048 # The ``project()`` command manages VERSION variables.
     CMP0054 # Only interpret ``if()`` arguments as variables or keywords when unquoted.
+    CMP0074 # ``find_package()`` uses ``<PackageName>_ROOT`` variables.
   )
     if(POLICY ${policy})
       cmake_policy(SET ${policy} NEW)
@@ -115,11 +116,9 @@ macro(set_fairmq_defaults)
   # Handle C++ standard level
   set(CMAKE_CXX_STANDARD_REQUIRED ON)
   if(NOT CMAKE_CXX_STANDARD)
-    set(CMAKE_CXX_STANDARD 11)
-  elseif(${CMAKE_CXX_STANDARD} LESS 11)
-    message(FATAL_ERROR "A minimum CMAKE_CXX_STANDARD of 11 is required.")
-  elseif(${CMAKE_CXX_STANDARD} GREATER 11)
-    message(WARNING "A CMAKE_CXX_STANDARD of ${CMAKE_CXX_STANDARD} (greater than 11) is not tested. Use on your own risk.")
+    set(CMAKE_CXX_STANDARD ${PROJECT_MIN_CXX_STANDARD})
+  elseif(${CMAKE_CXX_STANDARD} LESS ${PROJECT_MIN_CXX_STANDARD})
+    message(FATAL_ERROR "A minimum CMAKE_CXX_STANDARD of ${PROJECT_MIN_CXX_STANDARD} is required.")
   endif()
   set(CMAKE_CXX_EXTENSIONS OFF)
 
@@ -261,6 +260,8 @@ endfunction()
 
 # Configure/Install CMake package
 macro(install_cmake_package)
+  list(SORT PROJECT_PACKAGE_DEPENDENCIES)
+  list(SORT PROJECT_INTERFACE_PACKAGE_DEPENDENCIES)
   include(CMakePackageConfigHelpers)
   set(PACKAGE_INSTALL_DESTINATION
     ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}-${PROJECT_GIT_VERSION}
@@ -293,35 +294,48 @@ macro(install_cmake_package)
 endmacro()
 
 macro(find_package2 qualifier pkgname)
-  cmake_parse_arguments(ARGS "" "VERSION" "COMPONENTS" ${ARGN})
+  cmake_parse_arguments(ARGS "" "" "VERSION;COMPONENTS" ${ARGN})
 
   string(TOUPPER ${pkgname} pkgname_upper)
   set(old_CPP ${CMAKE_PREFIX_PATH})
   set(CMAKE_PREFIX_PATH ${${pkgname_upper}_ROOT} $ENV{${pkgname_upper}_ROOT} ${CMAKE_PREFIX_PATH})
+  unset(__version__)
+  if(ARGS_VERSION)
+    list(GET ARGS_VERSION 0 __version__)
+    list(LENGTH ARGS_VERSION __length__)
+    foreach(v IN LISTS ARGS_VERSION)
+      if(${v} VERSION_GREATER ${__version__})
+        set(__version__ ${v})
+      endif()
+    endforeach()
+  endif()
   if(ARGS_COMPONENTS)
-    find_package(${pkgname} ${ARGS_VERSION} QUIET COMPONENTS ${ARGS_COMPONENTS} ${ARGS_UNPARSED_ARGUMENTS})
+    list(REMOVE_DUPLICATES ARGS_COMPONENTS)
+    find_package(${pkgname} ${__version__} QUIET COMPONENTS ${ARGS_COMPONENTS} ${ARGS_UNPARSED_ARGUMENTS})
   else()
-    find_package(${pkgname} ${ARGS_VERSION} QUIET ${ARGS_UNPARSED_ARGUMENTS})
+    find_package(${pkgname} ${__version__} QUIET ${ARGS_UNPARSED_ARGUMENTS})
   endif()
   set(CMAKE_PREFIX_PATH ${old_CPP})
   unset(old_CPP)
 
   if(${pkgname}_FOUND)
     if(${qualifier} STREQUAL PRIVATE)
-      set(PROJECT_${pkgname}_VERSION ${ARGS_VERSION})
+      set(PROJECT_${pkgname}_VERSION ${__version__})
       set(PROJECT_${pkgname}_COMPONENTS ${ARGS_COMPONENTS})
       set(PROJECT_PACKAGE_DEPENDENCIES ${PROJECT_PACKAGE_DEPENDENCIES} ${pkgname})
     elseif(${qualifier} STREQUAL PUBLIC)
-      set(PROJECT_${pkgname}_VERSION ${ARGS_VERSION})
+      set(PROJECT_${pkgname}_VERSION ${__version__})
       set(PROJECT_${pkgname}_COMPONENTS ${ARGS_COMPONENTS})
       set(PROJECT_PACKAGE_DEPENDENCIES ${PROJECT_PACKAGE_DEPENDENCIES} ${pkgname})
-      set(PROJECT_INTERFACE_${pkgname}_VERSION ${ARGS_VERSION})
+      set(PROJECT_INTERFACE_${pkgname}_VERSION ${__version__})
       set(PROJECT_INTERFACE_${pkgname}_COMPONENTS ${ARGS_COMPONENTS})
       set(PROJECT_INTERFACE_PACKAGE_DEPENDENCIES ${PROJECT_INTERFACE_PACKAGE_DEPENDENCIES} ${pkgname})
     elseif(${qualifier} STREQUAL INTERFACE)
-      set(PROJECT_INTERFACE_${pkgname}_VERSION ${ARGS_VERSION})
+      set(PROJECT_INTERFACE_${pkgname}_VERSION ${__version__})
       set(PROJECT_INTERFACE_${pkgname}_COMPONENTS ${ARGS_COMPONENTS})
       set(PROJECT_INTERFACE_PACKAGE_DEPENDENCIES ${PROJECT_INTERFACE_PACKAGE_DEPENDENCIES} ${pkgname})
     endif()
   endif()
+
+  unset(__version__)
 endmacro()

cmake/FindAZMQ.cmake

@@ -0,0 +1,57 @@
+################################################################################
+#    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+#
+# ###########################
+# # Locate the AZMQ library #
+# ###########################
+#
+#
+# Usage:
+#
+#   find_package(AZMQ [version] [QUIET] [REQUIRED])
+#
+#
+# Defines the following variables:
+#
+#   AZMQ_FOUND - Found the ZeroMQ library
+#   AZMQ_INCLUDE_DIR (CMake cache) - Include directory
+#
+#
+# Accepts the following variables as hints for installation directories:
+#
+#   AZMQ_ROOT (CMake var, ENV var)
+#
+#
+# If the above variables are not defined, or if ZeroMQ could not be found there,
+# it will look for it in the system directories. Custom ZeroMQ installations
+# will always have priority over system ones.
+#
+
+if(NOT AZMQ_ROOT)
+  set(AZMQ_ROOT $ENV{AZMQ_ROOT})
+endif()
+
+find_path(AZMQ_INCLUDE_DIR
+  NAMES azmq/socket.hpp
+  HINTS ${AZMQ_ROOT} $ENV{AZMQ_ROOT}
+  PATH_SUFFIXES include
+  DOC "AZMQ include directory"
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(AZMQ
+    REQUIRED_VARS AZMQ_INCLUDE_DIR
+)
+
+if(AZMQ_FOUND AND NOT TARGET AZMQ::AZMQ)
+  add_library(AZMQ::AZMQ INTERFACE IMPORTED)
+  set_target_properties(AZMQ::AZMQ PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES ${AZMQ_INCLUDE_DIR}
+    INTERFACE_LINK_LIBRARIES "libzmq;Boost::boost;Boost::container;Boost::system"
+  )
+endif()

cmake/FindFlatBuffers.cmake

@@ -1,67 +0,0 @@
-################################################################################
-# Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
-#                                                                              #
-#              This software is distributed under the terms of the             #
-#              GNU Lesser General Public Licence (LGPL) version 3,             #
-#                  copied verbatim in the file "LICENSE"                       #
-################################################################################
-
-find_path(FLATBUFFERS_INCLUDE_DIR
-  NAMES flatbuffers/flatbuffers.h
-  HINTS ${FLATBUFFERS_ROOT} $ENV{FLATBUFFERS_ROOT}
-  PATH_SUFFIXES include
-)
-
-find_path(FLATBUFFERS_LIBRARY_DIR
-  NAMES libflatbuffers.a
-  HINTS ${FLATBUFFERS_ROOT} $ENV{FLATBUFFERS_ROOT}
-  PATH_SUFFIXES lib
-)
-
-find_library(FLATBUFFERS_STATIC_LIBRARY
-  NAMES libflatbuffers.a
-  HINTS ${FLATBUFFERS_ROOT} $ENV{FLATBUFFERS_ROOT}
-  PATH_SUFFIXES lib
-)
-
-find_path(FLATBUFFERS_BINARY_DIR
-  NAMES flatc
-  HINTS ${FLATBUFFERS_ROOT} $ENV{FLATBUFFERS_ROOT}
-  PATH_SUFFIXES bin
-)
-
-find_program(FLATBUFFERS_BINARY_FLATC
-  NAMES flatc
-  HINTS ${FLATBUFFERS_ROOT} $ENV{FLATBUFFERS_ROOT}
-  PATH_SUFFIXES bin
-)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(FlatBuffers
-  REQUIRED_VARS
-  FLATBUFFERS_INCLUDE_DIR
-  FLATBUFFERS_LIBRARY_DIR
-  FLATBUFFERS_BINARY_DIR
-)
-
-# idempotently import targets
-if(NOT TARGET FlatBuffers)
-  if(FLATBUFFERS_FOUND)
-    # import target
-    add_library(FlatBuffers STATIC IMPORTED)
-    set_target_properties(FlatBuffers PROPERTIES
-      INTERFACE_INCLUDE_DIRECTORIES ${FLATBUFFERS_INCLUDE_DIR}
-      IMPORTED_LOCATION ${FLATBUFFERS_STATIC_LIBRARY}
-    )
-  endif()
-endif()
-
-if(NOT TARGET FlatBuffers::flatc)
-  if(FLATBUFFERS_FOUND)
-  # import target
-  add_executable(FlatBuffers::flatc IMPORTED)
-  set_target_properties(FlatBuffers::flatc PROPERTIES
-    IMPORTED_LOCATION ${FLATBUFFERS_BINARY_FLATC}
-  )
-  endif()
-endif()

cmake/FindOFI.cmake

@@ -1,89 +0,0 @@
-################################################################################
-#    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
-#                                                                              #
-#              This software is distributed under the terms of the             #
-#              GNU Lesser General Public Licence (LGPL) version 3,             #
-#                  copied verbatim in the file "LICENSE"                       #
-################################################################################
-
-# According to the docs the modification of the PKG_CONFIG_PATH environment should
-# not be necessary, but it does not work otherwise.
-if(OFI_ROOT)
-  list(APPEND CMAKE_PREFIX_PATH "${OFI_ROOT}/lib/pkgconfig")
-  set(ENV{PKG_CONFIG_PATH} "${OFI_ROOT}/lib/pkgconfig:" $ENV{PKG_CONFIG_PATH})
-endif()
-
-if(ENV{OFI_ROOT})
-  list(APPEND CMAKE_PREFIX_PATH "$ENV{OFI_ROOT}/lib/pkgconfig")
-  set(ENV{PKG_CONFIG_PATH} "$ENV{OFI_ROOT}/lib/pkgconfig:" $ENV{PKG_CONFIG_PATH})
-endif()
-
-# This should be the default as of CMake 3.1, but it is not set. BUG? Also, it does not work
-set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH 1)
-find_package(PkgConfig QUIET)
-
-if(PKG_CONFIG_FOUND)
-  # Find include dir and dependencies from pkgconfig
-  pkg_check_modules(_OFI libfabric QUIET)
-
-  # Retrieve version from pkgconfig
-  execute_process(
-    COMMAND ${PKG_CONFIG_EXECUTABLE} libfabric --modversion
-    OUTPUT_VARIABLE OFI_VERSION
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-  )
-
-  # The IMPORTED_TARGET option of the pkg_check_modules() function is useless,
-  # so let's build it ourselves
-  find_library(OFI_LIBFABRIC
-    NAMES libfabric.so libfabric.dylib
-    HINTS ${OFI_ROOT} $ENV{OFI_ROOT}
-    PATH_SUFFIXES lib
-  )
-
-  # Just take the include dirs found by the PkgConfig module
-  set(OFI_INCLUDE_DIRS ${_OFI_INCLUDE_DIRS})
-
-  # Find fi_info command to be able to check required features of the OFI installation
-  find_program(OFI_INFO_EXECUTABLE
-    NAMES fi_info
-    HINTS ${OFI_ROOT} $ENV{OFI_ROOT}
-    PATH_SUFFIXES bin
-  )
-
-  # Detect ofi providers, they can be required via the COMPONENTS argument of find_package
-  if(OFI_INFO_EXECUTABLE)
-    execute_process(
-      COMMAND ${OFI_INFO_EXECUTABLE} -l
-      OUTPUT_VARIABLE output
-    )
-    string(REPLACE "\n" ";" lines ${output})
-    foreach(line IN LISTS lines)
-      string(REGEX
-        MATCH "^([a-zA-Z0-9_]+):"
-        found "${line}"
-      )
-      if(found)
-        string(TOLOWER "${CMAKE_MATCH_1}" provider)
-        set(OFI_fi_${provider}_FOUND TRUE)
-      endif()
-    endforeach()
-  endif()
-
-  # Check search result, check version constraints and print status
-  include(FindPackageHandleStandardArgs)
-  find_package_handle_standard_args(OFI
-    REQUIRED_VARS OFI_LIBFABRIC OFI_INCLUDE_DIRS OFI_INFO_EXECUTABLE
-    VERSION_VAR OFI_VERSION
-    HANDLE_COMPONENTS
-  )
-endif()
-
-if(NOT TARGET OFI::libfabric AND OFI_FOUND)
-  # Define an imported target
-  add_library(OFI::libfabric SHARED IMPORTED GLOBAL)
-  set_target_properties(OFI::libfabric PROPERTIES
-    IMPORTED_LOCATION ${OFI_LIBFABRIC}
-    INTERFACE_INCLUDE_DIRECTORIES ${OFI_INCLUDE_DIRS}
-  )
-endif()

cmake/FindPMIx.cmake

@@ -0,0 +1,67 @@
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+find_path(PMIx_INCLUDE_DIR
+  NAMES pmix.h
+  HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
+  PATH_SUFFIXES include
+)
+
+find_path(PMIx_LIBRARY_DIR
+  NAMES libpmix.dylib libpmix.so
+  HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
+  PATH_SUFFIXES lib lib64
+)
+
+find_library(PMIx_LIBRARY_SHARED
+  NAMES libpmix.dylib libpmix.so
+  HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
+  PATH_SUFFIXES lib lib64
+)
+
+find_file(PMIx_VERSION_FILE
+  NAMES pmix_version.h
+  HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
+  PATH_SUFFIXES include
+)
+
+file(READ "${PMIx_VERSION_FILE}" __version_raw)
+string(REGEX MATCH "#define PMIX_VERSION_MAJOR ([0-9]?)L?"
+  __version_major_raw "${__version_raw}"
+)
+set(PMIx_VERSION_MAJOR "${CMAKE_MATCH_1}")
+
+string(REGEX MATCH "#define PMIX_VERSION_MINOR ([0-9]?)L?"
+  __version_minor_raw "${__version_raw}"
+)
+set(PMIx_VERSION_MINOR "${CMAKE_MATCH_1}")
+
+string(REGEX MATCH "#define PMIX_VERSION_RELEASE ([0-9]?)L?"
+  __version_patch_raw "${__version_raw}"
+)
+set(PMIx_VERSION_PATCH "${CMAKE_MATCH_1}")
+
+set(PMIx_VERSION "${PMIx_VERSION_MAJOR}.${PMIx_VERSION_MINOR}.${PMIx_VERSION_PATCH}")
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(PMIx
+  REQUIRED_VARS
+  PMIx_INCLUDE_DIR
+  PMIx_LIBRARY_DIR
+  PMIx_LIBRARY_SHARED
+
+  VERSION_VAR PMIx_VERSION
+)
+
+if(NOT TARGET PMIx::libpmix AND PMIx_FOUND)
+  add_library(PMIx::libpmix SHARED IMPORTED)
+  set_target_properties(PMIx::libpmix PROPERTIES
+    IMPORTED_LOCATION ${PMIx_LIBRARY_SHARED}
+    INTERFACE_INCLUDE_DIRECTORIES ${PMIx_INCLUDE_DIR}
+  )
+endif()

cmake/GTestHelper.cmake

@@ -47,6 +47,8 @@
 #
 #
 
+include(GoogleTest)
+
 function(add_testsuite suitename)
     cmake_parse_arguments(testsuite
         ""
@@ -73,13 +75,23 @@ function(add_testsuite suitename)
         target_compile_definitions("${target}" PUBLIC ${testsuite_DEFINITIONS})
     endif()
 
-    add_test(NAME "${suitename}" WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMAND ${target})
+    # add_test(NAME "${suitename}" WORKING_DIRECTORY ${CMAKE_BINARY_DIR} COMMAND ${target})
     if(testsuite_TIMEOUT)
-        set_tests_properties("${suitename}" PROPERTIES TIMEOUT ${testsuite_TIMEOUT})
+        # set_tests_properties("${suitename}" PROPERTIES TIMEOUT ${testsuite_TIMEOUT})
+    else()
+        set(testsuite_TIMEOUT 10)
     endif()
     if(testsuite_RUN_SERIAL)
-        set_tests_properties("${suitename}" PROPERTIES RUN_SERIAL ${testsuite_RUN_SERIAL})
+        # set_tests_properties("${suitename}" PROPERTIES RUN_SERIAL ${testsuite_RUN_SERIAL})
+    else()
+        set(testsuite_RUN_SERIAL OFF)
     endif()
+    gtest_discover_tests(${target}
+      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+      TEST_PREFIX ${suitename}.
+      PROPERTIES RUN_SERIAL ${testsuite_RUN_SERIAL}
+                 TIMEOUT ${testsuite_TIMEOUT}
+    )
 
     list(APPEND ALL_TEST_TARGETS ${target})
     set(ALL_TEST_TARGETS ${ALL_TEST_TARGETS} PARENT_SCOPE)

docs/Device.md

@@ -31,14 +31,15 @@ The state machine can be querried and controlled via `GetCurrentStateName()` and
 If the device is running in interactive mode (default), states can be changed via keyboard input:
 
  - `'h'` - help
- - `'p'` - pause
  - `'r'` - run
  - `'s'` - stop
  - `'t'` - reset task
  - `'d'` - reset device
  - `'q'` - end
  - `'j'` - init task
- - `'i'` - init device
+ - `'i'` - initialize
+ - `'b'` - bind
+ - `'x'` - connect
 
 Without the interactive mode, for example for a run in background, two other control mechanisms are available:
 

examples/1-1/CMakeLists.txt

@@ -31,16 +31,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-1-1.sh.in ${CMAKE_CUR
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-1-1.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh)
 
-add_test(NAME Example-1-1-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh zeromq)
-set_tests_properties(Example-1-1-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+add_test(NAME Example.1-1.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh zeromq)
+set_tests_properties(Example.1-1.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-1-1-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh nanomsg)
-  set_tests_properties(Example-1-1-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+  add_test(NAME Example.1-1.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh nanomsg)
+  set_tests_properties(Example.1-1.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 endif()
 
-add_test(NAME Example-1-1-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh shmem)
-set_tests_properties(Example-1-1-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+add_test(NAME Example.1-1.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh shmem)
+set_tests_properties(Example.1-1.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 
 # install
 

examples/1-n-1/CMakeLists.txt

@@ -39,16 +39,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-1-n-1.json ${CMAKE_CURRENT_BINARY_
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-1-n-1.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh)
 
-add_test(NAME Example-1-n-1-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh zeromq)
-set_tests_properties(Example-1-n-1-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+add_test(NAME Example.1-n-1.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh zeromq)
+set_tests_properties(Example.1-n-1.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-1-n-1-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh nanomsg)
-  set_tests_properties(Example-1-n-1-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+  add_test(NAME Example.1-n-1.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh nanomsg)
+  set_tests_properties(Example.1-n-1.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 endif()
 
-add_test(NAME Example-1-n-1-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh shmem)
-set_tests_properties(Example-1-n-1-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
+add_test(NAME Example.1-n-1.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh shmem)
+set_tests_properties(Example.1-n-1.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received: ")
 
 # install
 

examples/CMakeLists.txt

@@ -16,5 +16,6 @@ add_subdirectory(multiple-channels)
 if(BUILD_NANOMSG_TRANSPORT)
   add_subdirectory(multiple-transports)
 endif()
+add_subdirectory(readout)
 add_subdirectory(region)
 add_subdirectory(req-rep)

examples/builtin-devices/CMakeLists.txt

@@ -14,27 +14,27 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-builtin-devices.sh.in
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-builtin-devices.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh)
 
-add_test(NAME Example-Builtin-Devices-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq)
-set_tests_properties(Example-Builtin-Devices-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+add_test(NAME Example.BuiltinDevices.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq)
+set_tests_properties(Example.BuiltinDevices.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-Builtin-Devices-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh nanomsg)
-  set_tests_properties(Example-Builtin-Devices-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+  add_test(NAME Example.BuiltinDevices.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh nanomsg)
+  set_tests_properties(Example.BuiltinDevices.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 endif()
 
-add_test(NAME Example-Builtin-Devices-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem)
-set_tests_properties(Example-Builtin-Devices-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+add_test(NAME Example.BuiltinDevices.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem)
+set_tests_properties(Example.BuiltinDevices.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
-add_test(NAME Example-Builtin-Devices-zeromq-multipart COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq true 2)
-set_tests_properties(Example-Builtin-Devices-zeromq-multipart PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+add_test(NAME Example.BuiltinDevices.multipart.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq true 2)
+set_tests_properties(Example.BuiltinDevices.multipart.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-Builtin-Devices-nanomsg-multipart COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh nanomsg true 2)
-  set_tests_properties(Example-Builtin-Devices-nanomsg-multipart PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+  add_test(NAME Example.BuiltinDevices.multipart.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh nanomsg true 2)
+  set_tests_properties(Example.BuiltinDevices.multipart.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 endif()
 
-add_test(NAME Example-Builtin-Devices-shmem-multipart COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem true 2)
-set_tests_properties(Example-Builtin-Devices-shmem-multipart PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
+add_test(NAME Example.BuiltinDevices.multipart.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem true 2)
+set_tests_properties(Example.BuiltinDevices.multipart.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
 
 # install
 

examples/copypush/CMakeLists.txt

@@ -32,16 +32,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-copypush.sh.in ${CMAK
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-copypush.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh)
 
-add_test(NAME Example-CopyPush-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh zeromq)
-set_tests_properties(Example-CopyPush-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
+add_test(NAME Example.CopyPush.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh zeromq)
+set_tests_properties(Example.CopyPush.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-CopyPush-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh nanomsg)
-  set_tests_properties(Example-CopyPush-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
+  add_test(NAME Example.CopyPush.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh nanomsg)
+  set_tests_properties(Example.CopyPush.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
 endif()
 
-add_test(NAME Example-CopyPush-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh shmem)
-set_tests_properties(Example-CopyPush-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
+add_test(NAME Example.CopyPush.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh shmem)
+set_tests_properties(Example.CopyPush.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message: ")
 
 # install
 

examples/dds/Processor.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

examples/dds/Sampler.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

examples/dds/Sink.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

examples/dds/ex-dds-hosts.cfg

@@ -2,6 +2,6 @@
 # source setup.sh
 @bash_end@
 
-sampler, username@localhost, , /path/to/dds-work-dir/, 1
-processor, username@localhost, , /path/to/dds-work-dir/, 10
-sink, username@localhost, , /path/to/dds-work-dir/, 1
+sampler, username@localhost, , /path/to/dds-work/, 1
+processor, username@localhost, , /path/to/dds-work/, 10
+sink, username@localhost, , /path/to/dds-work/, 1

examples/multipart/CMakeLists.txt

@@ -31,16 +31,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multipart.sh.in ${CMA
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multipart.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh)
 
-add_test(NAME Example-Multipart-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh zeromq)
-set_tests_properties(Example-Multipart-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 2 parts")
+add_test(NAME Example.Multipart.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh zeromq)
+set_tests_properties(Example.Multipart.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 2 parts")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-Multipart-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh nanomsg)
-  set_tests_properties(Example-Multipart-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 2 parts")
+  add_test(NAME Example.Multipart.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh nanomsg)
+  set_tests_properties(Example.Multipart.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 2 parts")
 endif()
 
-add_test(NAME Example-Multipart-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh shmem)
-set_tests_properties(Example-Multipart-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 2 parts")
+add_test(NAME Example.Multipart.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh shmem)
+set_tests_properties(Example.Multipart.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received message with 2 parts")
 
 # install
 

examples/multiple-channels/CMakeLists.txt

@@ -36,12 +36,12 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multiple-channels.sh.
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multiple-channels.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh)
 
-add_test(NAME Example-Multiple-Channels-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh zeromq)
-set_tests_properties(Example-Multiple-Channels-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
+add_test(NAME Example.MultipleChannels.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh zeromq)
+set_tests_properties(Example.MultipleChannels.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-Multiple-Channels-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh nanomsg)
-  set_tests_properties(Example-Multiple-Channels-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
+  add_test(NAME Example.MultipleChannels.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh nanomsg)
+  set_tests_properties(Example.MultipleChannels.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
 endif()
 
 # install

examples/multiple-channels/Sampler.cxx

@@ -41,7 +41,7 @@ void Sampler::Run()
 {
     FairMQPollerPtr poller(NewPoller("data", "broadcast"));
 
-    while (CheckCurrentState(RUNNING))
+    while (!NewStatePending())
     {
         poller->Poll(100);
 

examples/multiple-transports/CMakeLists.txt

@@ -35,8 +35,8 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multiple-transports.s
 # test
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multiple-transports.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-transports.sh)
-add_test(NAME Example-Multiple-Transports COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-transports.sh)
-set_tests_properties(Example-Multiple-Transports PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
+add_test(NAME Example.MultipleTransports COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-transports.sh)
+set_tests_properties(Example.MultipleTransports PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received messages from both sources.")
 
 # install
 

examples/multiple-transports/Sampler1.cxx

@@ -60,7 +60,7 @@ void Sampler1::ListenForAcks()
 {
     uint64_t numAcks = 0;
 
-    while (CheckCurrentState(RUNNING))
+    while (!NewStatePending())
     {
         FairMQMessagePtr ack(NewMessageFor("ack", 0));
         if (Receive(ack, "ack") < 0)

examples/readout/Builder.h

@@ -0,0 +1,40 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIRMQEXAMPLEREGIONBUILDER_H
+#define FAIRMQEXAMPLEREGIONBUILDER_H
+
+#include <atomic>
+
+#include "FairMQDevice.h"
+
+namespace example_region
+{
+
+class Builder : public FairMQDevice
+{
+  public:
+    Builder() {
+        OnData("data1", &Builder::HandleData);
+    }
+    virtual ~Builder() {}
+
+  protected:
+    bool HandleData(FairMQMessagePtr& msg, int /*index*/)
+    {
+        if (Send(msg, "data2") < 0) {
+            return false;
+        }
+
+        return true;
+    }
+};
+
+} // namespace example_region
+
+#endif /* FAIRMQEXAMPLEREGIONBUILDER_H */

examples/readout/CMakeLists.txt

@@ -0,0 +1,55 @@
+ ################################################################################
+ #    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+ #                                                                              #
+ #              This software is distributed under the terms of the             #
+ #              GNU Lesser General Public Licence (LGPL) version 3,             #
+ #                  copied verbatim in the file "LICENSE"                       #
+ ################################################################################
+
+add_library(ExampleReadoutLib STATIC
+    "Sampler.cxx"
+    "Sampler.h"
+    "Builder.h"
+    "Sink.cxx"
+    "Sink.h"
+)
+
+target_link_libraries(ExampleReadoutLib PUBLIC FairMQ)
+
+add_executable(fairmq-ex-readout-sampler runSampler.cxx)
+target_link_libraries(fairmq-ex-readout-sampler PRIVATE ExampleReadoutLib)
+
+add_executable(fairmq-ex-readout-builder runBuilder.cxx)
+target_link_libraries(fairmq-ex-readout-builder PRIVATE ExampleReadoutLib)
+
+add_executable(fairmq-ex-readout-sink runSink.cxx)
+target_link_libraries(fairmq-ex-readout-sink PRIVATE ExampleReadoutLib)
+
+add_custom_target(ExampleReadout DEPENDS fairmq-ex-readout-sampler fairmq-ex-readout-builder fairmq-ex-readout-sink)
+
+set(EX_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR})
+set(FAIRMQ_BIN_DIR ${CMAKE_BINARY_DIR}/fairmq)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-readout.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout.sh)
+
+# install
+
+install(
+    TARGETS
+    fairmq-ex-readout-sampler
+    fairmq-ex-readout-builder
+    fairmq-ex-readout-sink
+
+    LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
+    RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
+)
+
+# configure run script with different executable paths for build and for install directories
+set(EX_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
+set(FAIRMQ_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR}/fairmq)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-readout.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout.sh_install)
+
+install(
+    PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout.sh_install
+    DESTINATION ${PROJECT_INSTALL_BINDIR}
+    RENAME fairmq-start-ex-readout.sh
+)

examples/readout/Sampler.cxx

@@ -0,0 +1,86 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+/**
+ * Sampler.cpp
+ *
+ * @since 2014-10-10
+ * @author A. Rybalchenko
+ */
+
+#include "Sampler.h"
+
+#include <thread>
+
+using namespace std;
+
+namespace example_region
+{
+
+Sampler::Sampler()
+    : fMsgSize(10000)
+    , fMaxIterations(0)
+    , fNumIterations(0)
+    , fRegion(nullptr)
+    , fNumUnackedMsgs(0)
+{
+}
+
+void Sampler::InitTask()
+{
+    fMsgSize = fConfig->GetValue<int>("msg-size");
+    fMaxIterations = fConfig->GetValue<uint64_t>("max-iterations");
+
+    fRegion = FairMQUnmanagedRegionPtr(NewUnmanagedRegion(10000000,
+                                                          [this](void* /*data*/, size_t /*size*/, void* /*hint*/) { // callback to be called when message buffers no longer needed by transport
+                                                              --fNumUnackedMsgs;
+                                                              if (fMaxIterations > 0)
+                                                              {
+                                                                  LOG(debug) << "Received ack";
+                                                              }
+                                                          }));
+}
+
+bool Sampler::ConditionalRun()
+{
+    FairMQMessagePtr msg(NewMessage(fRegion, // region
+                                    fRegion->GetData(), // ptr within region
+                                    fMsgSize, // offset from ptr
+                                    nullptr // hint
+                                    ));
+
+    if (Send(msg, "data1", 0) > 0)
+    {
+        ++fNumUnackedMsgs;
+
+        if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations)
+        {
+            LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
+            return false;
+        }
+    }
+
+    return true;
+}
+
+void Sampler::ResetTask()
+{
+    // if not all messages acknowledged, wait for a bit. But only once, since receiver could be already dead.
+    if (fNumUnackedMsgs != 0)
+    {
+        LOG(debug) << "waiting for all acknowledgements... (" << fNumUnackedMsgs << ")";
+        this_thread::sleep_for(chrono::milliseconds(500));
+        LOG(debug) << "done, still unacked: " << fNumUnackedMsgs;
+    }
+    fRegion.reset();
+}
+
+Sampler::~Sampler()
+{
+}
+
+} // namespace example_region

examples/readout/Sampler.h

@@ -0,0 +1,46 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+/**
+ * Sampler.h
+ *
+ * @since 2014-10-10
+ * @author A. Rybalchenko
+ */
+
+#ifndef FAIRMQEXAMPLEREGIONSAMPLER_H
+#define FAIRMQEXAMPLEREGIONSAMPLER_H
+
+#include <atomic>
+
+#include "FairMQDevice.h"
+
+namespace example_region
+{
+
+class Sampler : public FairMQDevice
+{
+  public:
+    Sampler();
+    virtual ~Sampler();
+
+  protected:
+    virtual void InitTask();
+    virtual bool ConditionalRun();
+    virtual void ResetTask();
+
+  private:
+    int fMsgSize;
+    uint64_t fMaxIterations;
+    uint64_t fNumIterations;
+    FairMQUnmanagedRegionPtr fRegion;
+    std::atomic<uint64_t> fNumUnackedMsgs;
+};
+
+} // namespace example_region
+
+#endif /* FAIRMQEXAMPLEREGIONSAMPLER_H */

examples/readout/Sink.cxx

@@ -0,0 +1,56 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+/**
+ * Sink.cxx
+ *
+ * @since 2014-10-10
+ * @author A. Rybalchenko
+ */
+
+#include "Sink.h"
+
+using namespace std;
+
+namespace example_region
+{
+
+Sink::Sink()
+    : fMaxIterations(0)
+    , fNumIterations(0)
+{
+}
+
+void Sink::InitTask()
+{
+    // Get the fMaxIterations value from the command line options (via fConfig)
+    fMaxIterations = fConfig->GetValue<uint64_t>("max-iterations");
+}
+
+void Sink::Run()
+{
+    FairMQChannel& dataInChannel = fChannels.at("data").at(0);
+
+    while (!NewStatePending())
+    {
+        FairMQMessagePtr msg(dataInChannel.Transport()->CreateMessage());
+        dataInChannel.Receive(msg);
+        // void* ptr = msg->GetData();
+
+        if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations)
+        {
+            LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
+            break;
+        }
+    }
+}
+
+Sink::~Sink()
+{
+}
+
+} // namespace example_region

examples/readout/Sink.h

@@ -0,0 +1,42 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+/**
+ * Sink.h
+ *
+ * @since 2014-10-10
+ * @author A. Rybalchenko
+ */
+
+#ifndef FAIRMQEXAMPLEREGIONSINK_H
+#define FAIRMQEXAMPLEREGIONSINK_H
+
+#include <string>
+
+#include "FairMQDevice.h"
+
+namespace example_region
+{
+
+class Sink : public FairMQDevice
+{
+  public:
+    Sink();
+    virtual ~Sink();
+
+  protected:
+    virtual void Run();
+    virtual void InitTask();
+
+  private:
+    uint64_t fMaxIterations;
+    uint64_t fNumIterations;
+};
+
+} // namespace example_region
+
+#endif /* FAIRMQEXAMPLEREGIONSINK_H */

examples/readout/fairmq-start-ex-readout.sh.in

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
+
+msgSize="2000000"
+
+if [[ $1 =~ ^[0-9]+$ ]]; then
+    msgSize=$1
+fi
+
+SAMPLER="fairmq-ex-readout-sampler"
+SAMPLER+=" --id sampler1"
+SAMPLER+=" --severity debug"
+SAMPLER+=" --transport shmem"
+SAMPLER+=" --msg-size $msgSize"
+# SAMPLER+=" --rate 10"
+SAMPLER+=" --channel-config name=data1,type=pair,method=bind,address=tcp://127.0.0.1:7777,transport=shmem"
+xterm -geometry 80x23+0+0 -hold -e @EX_BIN_DIR@/$SAMPLER &
+
+BUILDER="fairmq-ex-readout-builder"
+BUILDER+=" --id builder1"
+BUILDER+=" --severity debug"
+BUILDER+=" --channel-config name=data1,type=pair,method=connect,address=tcp://127.0.0.1:7777,transport=shmem"
+BUILDER+="                  name=data2,type=pair,method=connect,address=tcp://127.0.0.1:7778,transport=ofi"
+xterm -geometry 80x23+500+0 -hold -e @EX_BIN_DIR@/$BUILDER &
+
+SINK="fairmq-ex-readout-sink"
+SINK+=" --id sink1"
+SINK+=" --severity debug"
+SINK+=" --ofi-size-hint $msgSize"
+SINK+=" --channel-config name=data,type=pair,method=bind,address=tcp://127.0.0.1:7778,transport=ofi"
+xterm -geometry 80x23+1000+0 -hold -e @EX_BIN_DIR@/$SINK &

examples/readout/runBuilder.cxx

@@ -0,0 +1,20 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "Builder.h"
+
+namespace bpo = boost::program_options;
+
+void addCustomOptions(bpo::options_description& /* options */)
+{}
+
+FairMQDevicePtr getDevice(const FairMQProgOptions& /*config*/)
+{
+    return new example_region::Builder();
+}

examples/readout/runSampler.cxx

@@ -0,0 +1,24 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "Sampler.h"
+
+namespace bpo = boost::program_options;
+
+void addCustomOptions(bpo::options_description& options)
+{
+    options.add_options()
+        ("msg-size", bpo::value<int>()->default_value(1000), "Message size in bytes")
+        ("max-iterations", bpo::value<uint64_t>()->default_value(0), "Maximum number of iterations of Run/ConditionalRun/OnData (0 - infinite)");
+}
+
+FairMQDevicePtr getDevice(const FairMQProgOptions& /*config*/)
+{
+    return new example_region::Sampler();
+}

examples/readout/runSink.cxx

@@ -0,0 +1,23 @@
+/********************************************************************************
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "runFairMQDevice.h"
+#include "Sink.h"
+
+namespace bpo = boost::program_options;
+
+void addCustomOptions(bpo::options_description& options)
+{
+    options.add_options()
+        ("max-iterations", bpo::value<uint64_t>()->default_value(0), "Maximum number of iterations of Run/ConditionalRun/OnData (0 - infinite)");
+}
+
+FairMQDevicePtr getDevice(const FairMQProgOptions& /*config*/)
+{
+    return new example_region::Sink();
+}

examples/region/CMakeLists.txt

@@ -18,7 +18,6 @@ target_link_libraries(ExampleRegionLib PUBLIC FairMQ)
 add_executable(fairmq-ex-region-sampler runSampler.cxx)
 target_link_libraries(fairmq-ex-region-sampler PRIVATE ExampleRegionLib)
 
-
 add_executable(fairmq-ex-region-sink runSink.cxx)
 target_link_libraries(fairmq-ex-region-sink PRIVATE ExampleRegionLib)
 
@@ -32,16 +31,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-region.sh.in ${CMAKE_
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-region.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh)
 
-add_test(NAME Example-Region-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh zeromq)
-set_tests_properties(Example-Region-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
+add_test(NAME Example.Region.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh zeromq)
+set_tests_properties(Example.Region.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-Region-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh nanomsg)
-  set_tests_properties(Example-Region-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
+  add_test(NAME Example.Region.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh nanomsg)
+  set_tests_properties(Example.Region.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
 endif()
 
-add_test(NAME Example-Region-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh shmem)
-set_tests_properties(Example-Region-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
+add_test(NAME Example.Region.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh shmem)
+set_tests_properties(Example.Region.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received ack")
 
 # install
 

examples/region/Sink.cxx

@@ -35,7 +35,7 @@ void Sink::Run()
 {
     FairMQChannel& dataInChannel = fChannels.at("data").at(0);
 
-    while (CheckCurrentState(RUNNING))
+    while (!NewStatePending())
     {
         FairMQMessagePtr msg(dataInChannel.Transport()->CreateMessage());
         dataInChannel.Receive(msg);

examples/req-rep/CMakeLists.txt

@@ -32,16 +32,16 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-req-rep.sh.in ${CMAKE
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-req-rep.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh)
 
-add_test(NAME Example-ReqRep-zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh zeromq)
-set_tests_properties(Example-ReqRep-zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
+add_test(NAME Example.ReqRep.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh zeromq)
+set_tests_properties(Example.ReqRep.zeromq PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
 
 if(BUILD_NANOMSG_TRANSPORT)
-  add_test(NAME Example-ReqRep-nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh nanomsg)
-  set_tests_properties(Example-ReqRep-nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
+  add_test(NAME Example.ReqRep.nanomsg COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh nanomsg)
+  set_tests_properties(Example.ReqRep.nanomsg PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
 endif()
 
-add_test(NAME Example-ReqRep-shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh shmem)
-set_tests_properties(Example-ReqRep-shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
+add_test(NAME Example.ReqRep.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh shmem)
+set_tests_properties(Example.ReqRep.shmem PROPERTIES TIMEOUT "30" RUN_SERIAL true PASS_REGULAR_EXPRESSION "Received reply from server: ")
 
 # install
 

fairmq/CMakeLists.txt

@@ -12,24 +12,8 @@
 if(BUILD_DDS_PLUGIN)
   add_subdirectory(plugins/DDS)
 endif()
-
-
-############################
-# preprocessor definitions #
-############################
-if(BUILD_NANOMSG_TRANSPORT)
-  add_definitions(-DBUILD_NANOMSG_TRANSPORT)
-endif()
-if(BUILD_OFI_TRANSPORT)
-  add_definitions(-DBUILD_OFI_TRANSPORT)
-endif()
-
-
-##################
-# subdirectories #
-##################
-if(BUILD_OFI_TRANSPORT)
-  add_subdirectory(ofi)
+if(BUILD_PMIX_PLUGIN)
+  add_subdirectory(plugins/PMIx)
 endif()
 
 
@@ -47,7 +31,7 @@ set(FAIRMQ_PUBLIC_HEADER_FILES
   FairMQPoller.h
   FairMQUnmanagedRegion.h
   FairMQSocket.h
-  FairMQStateMachine.h
+  StateMachine.h
   FairMQTransportFactory.h
   MemoryResources.h
   MemoryResourceTools.h
@@ -80,7 +64,6 @@ set(FAIRMQ_PRIVATE_HEADER_FILES
   options/FairProgOptionsHelper.h
   plugins/Builtin.h
   plugins/Control.h
-  StateMachine.h
   shmem/FairMQMessageSHM.h
   shmem/FairMQPollerSHM.h
   shmem/FairMQUnmanagedRegionSHM.h
@@ -127,7 +110,7 @@ set(FAIRMQ_SOURCE_FILES
   FairMQMessage.cxx
   FairMQPoller.cxx
   FairMQSocket.cxx
-  FairMQStateMachine.cxx
+  StateMachine.cxx
   FairMQTransportFactory.cxx
   devices/FairMQBenchmarkSampler.cxx
   devices/FairMQMerger.cxx
@@ -141,7 +124,6 @@ set(FAIRMQ_SOURCE_FILES
   PluginManager.cxx
   PluginServices.cxx
   plugins/Control.cxx
-  StateMachine.cxx
   shmem/FairMQMessageSHM.cxx
   shmem/FairMQPollerSHM.cxx
   shmem/FairMQUnmanagedRegionSHM.cxx
@@ -207,6 +189,19 @@ if(FAST_BUILD)
   set_target_properties(${_target} PROPERTIES OUTPUT_NAME FairMQ)
 endif()
 
+
+############################
+# preprocessor definitions #
+############################
+target_compile_definitions(${_target} PUBLIC BOOST_ERROR_CODE_HEADER_ONLY)
+if(BUILD_NANOMSG_TRANSPORT)
+  target_compile_definitions(${_target} PRIVATE BUILD_NANOMSG_TRANSPORT)
+endif()
+if(BUILD_OFI_TRANSPORT)
+  target_compile_definitions(${_target} PRIVATE BUILD_OFI_TRANSPORT)
+endif()
+
+
 #######################
 # include directories #
 #######################
@@ -226,7 +221,11 @@ if(BUILD_NANOMSG_TRANSPORT)
   set(NANOMSG_DEPS nanomsg msgpackc-cxx)
 endif()
 if(BUILD_OFI_TRANSPORT)
-  set(OFI_DEPS OFI::libfabric protobuf::libprotobuf $<TARGET_OBJECTS:OfiTransport>)
+  set(OFI_DEPS
+    asiofi::asiofi
+    Boost::container
+    AZMQ::AZMQ
+  )
 endif()
 set(optional_deps ${NANOMSG_DEPS} ${OFI_DEPS})
 if(optional_deps)
@@ -243,12 +242,8 @@ target_link_libraries(${_target}
   $<$<PLATFORM_ID:Linux>:rt>
   Boost::boost
   Boost::program_options
-  Boost::thread
-  Boost::system
   Boost::filesystem
   Boost::regex
-  Boost::date_time
-  Boost::signals
   FairLogger::FairLogger
 
   PRIVATE # only libFairMQ links against private dependencies

fairmq/DeviceRunner.cxx

@@ -74,15 +74,14 @@ auto DeviceRunner::Run() -> int
     fDevice->RegisterChannelEndpoints();
     if (fConfig.Count("print-channels")) {
         fDevice->PrintRegisteredChannels();
-        fDevice->ChangeState(FairMQDevice::END);
+        fDevice->ChangeState(fair::mq::Transition::End);
         return 0;
     }
 
     // Handle --version
     if (fConfig.Count("version")) {
         std::cout << "User device version: " << fDevice->GetVersion() << std::endl;
-        std::cout << "FAIRMQ_INTERFACE_VERSION: " << FAIRMQ_INTERFACE_VERSION << std::endl;
-        fDevice->ChangeState(FairMQDevice::END);
+        fDevice->ChangeState(fair::mq::Transition::End);
         return 0;
     }
 

fairmq/FairMQDevice.cxx

@@ -27,6 +27,34 @@
 
 using namespace std;
 
+static map<fair::mq::Transition, fair::mq::State> backwardsCompatibilityWaitForEndOfStateHelper =
+{
+    { fair::mq::Transition::InitDevice,   fair::mq::State::InitializingDevice },
+    { fair::mq::Transition::CompleteInit, fair::mq::State::Initialized },
+    { fair::mq::Transition::Bind,         fair::mq::State::Bound },
+    { fair::mq::Transition::Connect,      fair::mq::State::DeviceReady },
+    { fair::mq::Transition::InitTask,     fair::mq::State::Ready },
+    { fair::mq::Transition::Run,          fair::mq::State::Ready },
+    { fair::mq::Transition::Stop,         fair::mq::State::Ready },
+    { fair::mq::Transition::ResetTask,    fair::mq::State::DeviceReady },
+    { fair::mq::Transition::ResetDevice,  fair::mq::State::Idle }
+};
+
+static map<int, fair::mq::Transition> backwardsCompatibilityChangeStateHelper =
+{
+    { FairMQDevice::Event::INIT_DEVICE, fair::mq::Transition::InitDevice },
+    { FairMQDevice::Event::internal_DEVICE_READY, fair::mq::Transition::Auto },
+    { FairMQDevice::Event::INIT_TASK, fair::mq::Transition::InitTask },
+    { FairMQDevice::Event::internal_READY, fair::mq::Transition::Auto },
+    { FairMQDevice::Event::RUN, fair::mq::Transition::Run },
+    { FairMQDevice::Event::STOP, fair::mq::Transition::Stop },
+    { FairMQDevice::Event::RESET_TASK, fair::mq::Transition::ResetTask },
+    { FairMQDevice::Event::RESET_DEVICE, fair::mq::Transition::ResetDevice },
+    { FairMQDevice::Event::internal_IDLE, fair::mq::Transition::Auto },
+    { FairMQDevice::Event::END, fair::mq::Transition::End },
+    { FairMQDevice::Event::ERROR_FOUND, fair::mq::Transition::ErrorFound }
+};
+
 FairMQDevice::FairMQDevice()
     : FairMQDevice(nullptr, {0, 0, 0})
 {
@@ -54,7 +82,10 @@ FairMQDevice::FairMQDevice(FairMQProgOptions* config, const fair::mq::tools::Ver
     , fInternalConfig(config ? nullptr : fair::mq::tools::make_unique<FairMQProgOptions>())
     , fConfig(config ? config : fInternalConfig.get())
     , fId()
-    , fDefaultTransportType(fair::mq::Transport::DEFAULT)
+    , fDefaultTransportType(fair::mq::Transport::ZMQ)
+    , fStateMachine()
+    , fUninitializedBindingChannels()
+    , fUninitializedConnectingChannels()
     , fDataCallbacks(false)
     , fMsgInputs()
     , fMultipartInputs()
@@ -65,18 +96,107 @@ FairMQDevice::FairMQDevice(FairMQProgOptions* config, const fair::mq::tools::Ver
     , fMultitransportProceed(false)
     , fVersion(version)
     , fRate(0.)
+    , fMaxRunRuntimeInS(0)
     , fRawCmdLineArgs()
-    , fInterrupted(false)
-    , fInterruptedCV()
-    , fInterruptedMtx()
-    , fRateLogging(true)
 {
+    SubscribeToNewTransition("device", [&](fair::mq::Transition transition) {
+        LOG(trace) << "device notified on new transition: " << transition;
+
+        switch (transition) {
+            case fair::mq::Transition::Stop:
+                UnblockTransports();
+                break;
+            default:
+                break;
+        }
+    });
+
+    fStateMachine.HandleStates([&](fair::mq::State state) {
+        LOG(trace) << "device notified on new state: " << state;
+
+        {
+            lock_guard<mutex> lock(fStatesMtx);
+            fStates.push(state);
+        }
+        fStatesCV.notify_one();
+
+        switch (state) {
+            case fair::mq::State::InitializingDevice:
+                InitWrapper();
+                break;
+            case fair::mq::State::Binding:
+                BindWrapper();
+                break;
+            case fair::mq::State::Connecting:
+                ConnectWrapper();
+                break;
+            case fair::mq::State::InitializingTask:
+                InitTaskWrapper();
+                break;
+            case fair::mq::State::Running:
+                RunWrapper();
+                break;
+            case fair::mq::State::ResettingTask:
+                ResetTaskWrapper();
+                break;
+            case fair::mq::State::ResettingDevice:
+                ResetWrapper();
+                break;
+            case fair::mq::State::Exiting:
+                Exit();
+                break;
+            default:
+                LOG(trace) << "device notified on new state without a matching handler: " << state;
+                break;
+        }
+    });
+
+    fStateMachine.Start();
+}
+
+fair::mq::State FairMQDevice::WaitForNextState()
+{
+    unique_lock<mutex> lock(fStatesMtx);
+    while (fStates.empty()) {
+        fStatesCV.wait_for(lock, chrono::milliseconds(50));
+    }
+
+    auto result = fStates.front();
+
+    if (result == fair::mq::State::Error) {
+        throw DeviceStateError("Device transitioned to error state.");
+    }
+
+    fStates.pop();
+
+    return result;
+}
+
+void FairMQDevice::WaitForState(fair::mq::State state)
+{
+    while (WaitForNextState() != state) {}
+}
+
+bool FairMQDevice::ChangeState(const int transition)
+{
+    return ChangeState(backwardsCompatibilityChangeStateHelper.at(transition));
+}
+
+void FairMQDevice::WaitForEndOfState(fair::mq::Transition transition)
+{
+    WaitForState(backwardsCompatibilityWaitForEndOfStateHelper.at(transition));
 }
 
 void FairMQDevice::InitWrapper()
 {
+    fStateMachine.WaitForPendingState();
+
     fId = fConfig->GetValue<string>("id");
+
+    Init();
+
     fRate = fConfig->GetValue<float>("rate");
+    fMaxRunRuntimeInS = fConfig->GetValue<uint64_t>("max-run-time");
 
     try {
         fDefaultTransportType = fair::mq::TransportTypes.at(fConfig->GetValue<string>("transport"));
@@ -96,13 +216,9 @@ void FairMQDevice::InitWrapper()
         }
     }
 
-    LOG(debug) << "Requesting '" << fair::mq::TransportNames.at(fDefaultTransportType) << "' as default transport for the device";
+    LOG(debug) << "Setting '" << fair::mq::TransportNames.at(fDefaultTransportType) << "' as default transport for the device";
     fTransportFactory = AddTransport(fDefaultTransportType);
 
-    // Containers to store the uninitialized channels.
-    vector<FairMQChannel*> uninitializedBindingChannels;
-    vector<FairMQChannel*> uninitializedConnectingChannels;
-
     string networkInterface = fConfig->GetValue<string>("network-interface");
 
     // Fill the uninitialized channel containers
@@ -113,13 +229,8 @@ void FairMQDevice::InitWrapper()
             vi.fName = fair::mq::tools::ToString(mi.first, "[", subChannelIndex, "]");
 
             // set channel transport
-            if (vi.fTransportType == fair::mq::Transport::DEFAULT || vi.fTransportType == fTransportFactory->GetType()) {
-                LOG(debug) << vi.fName << ": using default transport";
-                vi.InitTransport(fTransportFactory);
-            } else {
-                LOG(debug) << vi.fName << ": channel transport (" << fair::mq::TransportNames.at(fDefaultTransportType) << ") overriden to " << fair::mq::TransportNames.at(vi.fTransportType);
-                vi.InitTransport(AddTransport(vi.fTransportType));
-            }
+            LOG(debug) << "Initializing transport for channel " << vi.fName << ": " << fair::mq::TransportNames.at(vi.fTransportType);
+            vi.InitTransport(AddTransport(vi.fTransportType));
 
             if (vi.fMethod == "bind") {
                 // if binding address is not specified, try getting it from the configured network interface
@@ -131,13 +242,13 @@ void FairMQDevice::InitWrapper()
                     vi.fAddress = "tcp://" + fair::mq::tools::getInterfaceIP(networkInterface) + ":1";
                 }
                 // fill the uninitialized list
-                uninitializedBindingChannels.push_back(&vi);
+                fUninitializedBindingChannels.push_back(&vi);
             } else if (vi.fMethod == "connect") {
                 // fill the uninitialized list
-                uninitializedConnectingChannels.push_back(&vi);
+                fUninitializedConnectingChannels.push_back(&vi);
             } else if (vi.fAddress.find_first_of("@+>") != string::npos) {
                 // fill the uninitialized list
-                uninitializedConnectingChannels.push_back(&vi);
+                fUninitializedConnectingChannels.push_back(&vi);
             } else {
                 LOG(error) << "Cannot update configuration. Socket method (bind/connect) for channel '" << vi.fName << "' not specified.";
                 throw runtime_error(fair::mq::tools::ToString("Cannot update configuration. Socket method (bind/connect) for channel ", vi.fName, " not specified."));
@@ -147,17 +258,27 @@ void FairMQDevice::InitWrapper()
         }
     }
 
+    // ChangeState(fair::mq::Transition::Auto);
+}
+
+void FairMQDevice::BindWrapper()
+{
     // Bind channels. Here one run is enough, because bind settings should be available locally
     // If necessary this could be handled in the same way as the connecting channels
-    AttachChannels(uninitializedBindingChannels);
+    AttachChannels(fUninitializedBindingChannels);
 
-    if (!uninitializedBindingChannels.empty()) {
-        LOG(error) << uninitializedBindingChannels.size() << " of the binding channels could not initialize. Initial configuration incomplete.";
-        throw runtime_error(fair::mq::tools::ToString(uninitializedBindingChannels.size(), " of the binding channels could not initialize. Initial configuration incomplete."));
+    if (!fUninitializedBindingChannels.empty()) {
+        LOG(error) << fUninitializedBindingChannels.size() << " of the binding channels could not initialize. Initial configuration incomplete.";
+        throw runtime_error(fair::mq::tools::ToString(fUninitializedBindingChannels.size(), " of the binding channels could not initialize. Initial configuration incomplete."));
     }
 
-    CallStateChangeCallbacks(INITIALIZING_DEVICE);
+    Bind();
 
+    ChangeState(fair::mq::Transition::Auto);
+}
+
+void FairMQDevice::ConnectWrapper()
+{
     int initializationTimeoutInS = fConfig->GetValue<int>("initialization-timeout");
 
     // go over the list of channels until all are initialized (and removed from the uninitialized list)
@@ -165,12 +286,12 @@ void FairMQDevice::InitWrapper()
     auto sleepTimeInMS = 50;
     auto maxAttempts = initializationTimeoutInS * 1000 / sleepTimeInMS;
     // first attempt
-    AttachChannels(uninitializedConnectingChannels);
+    AttachChannels(fUninitializedConnectingChannels);
     // if not all channels could be connected, update their address values from config and retry
-    while (!uninitializedConnectingChannels.empty()) {
+    while (!fUninitializedConnectingChannels.empty()) {
         this_thread::sleep_for(chrono::milliseconds(sleepTimeInMS));
 
-        for (auto& chan : uninitializedConnectingChannels) {
+        for (auto& chan : fUninitializedConnectingChannels) {
             string key{"chans." + chan->GetChannelPrefix() + "." + chan->GetChannelIndex() + ".address"};
             string newAddress = fConfig->GetValue<string>(key);
             if (newAddress != chan->GetAddress()) {
@@ -183,20 +304,16 @@ void FairMQDevice::InitWrapper()
             throw runtime_error(fair::mq::tools::ToString("could not connect all channels after ", initializationTimeoutInS, " attempts"));
         }
 
-        AttachChannels(uninitializedConnectingChannels);
+        AttachChannels(fUninitializedConnectingChannels);
     }
 
-    Init();
-
     if (fChannels.empty()) {
         LOG(warn) << "No channels created after finishing initialization";
     }
 
-    ChangeState(internal_DEVICE_READY);
-}
+    Connect();
 
-void FairMQDevice::Init()
-{
+    ChangeState(fair::mq::Transition::Auto);
 }
 
 void FairMQDevice::AttachChannels(vector<FairMQChannel*>& chans)
@@ -295,15 +412,9 @@ bool FairMQDevice::AttachChannel(FairMQChannel& chan)
 
 void FairMQDevice::InitTaskWrapper()
 {
-    CallStateChangeCallbacks(INITIALIZING_TASK);
-
     InitTask();
 
-    ChangeState(internal_READY);
-}
-
-void FairMQDevice::InitTask()
-{
+    ChangeState(fair::mq::Transition::Auto);
 }
 
 bool FairMQDevice::SortSocketsByAddress(const FairMQChannel &lhs, const FairMQChannel &rhs)
@@ -334,21 +445,13 @@ void FairMQDevice::SortChannel(const string& name, const bool reindex)
 
 void FairMQDevice::RunWrapper()
 {
-    CallStateChangeCallbacks(RUNNING);
-
     LOG(info) << "DEVICE: Running...";
 
     // start the rate logger thread
-    fRateLogging = true;
     future<void> rateLogger = async(launch::async, &FairMQDevice::LogSocketRates, this);
 
     // notify transports to resume transfers
-    {
-        lock_guard<mutex> guard(fInterruptedMtx);
-        fInterrupted = false;
-    }
-    for (auto& t : fTransports)
-    {
+    for (auto& t : fTransports) {
         t.second->Resume();
     }
 
@@ -356,50 +459,43 @@ void FairMQDevice::RunWrapper()
         PreRun();
 
         // process either data callbacks or ConditionalRun/Run
-        if (fDataCallbacks)
-        {
+        if (fDataCallbacks) {
             // if only one input channel, do lightweight handling without additional polling.
-            if (fInputChannelKeys.size() == 1 && fChannels.at(fInputChannelKeys.at(0)).size() == 1)
-            {
+            if (fInputChannelKeys.size() == 1 && fChannels.at(fInputChannelKeys.at(0)).size() == 1) {
                 HandleSingleChannelInput();
-            }
-            else // otherwise do full handling with polling
-            {
+            } else {// otherwise do full handling with polling
                 HandleMultipleChannelInput();
             }
-        }
-        else
-        {
+        } else {
             fair::mq::tools::RateLimiter rateLimiter(fRate);
 
-            while (CheckCurrentState(RUNNING) && ConditionalRun())
-            {
-                if (fRate > 0.001)
-                {
+            while (!NewStatePending() && ConditionalRun()) {
+                if (fRate > 0.001) {
                     rateLimiter.maybe_sleep();
                 }
             }
 
             Run();
         }
+
+        // if Run() exited and the state is still RUNNING, transition to READY.
+        if (!NewStatePending()) {
+            UnblockTransports();
+            ChangeState(fair::mq::Transition::Stop);
+        }
+
+        PostRun();
+
     } catch (const out_of_range& oor) {
         LOG(error) << "out of range: " << oor.what();
         LOG(error) << "incorrect/incomplete channel configuration?";
-        fRateLogging = false;
+        ChangeState(fair::mq::Transition::ErrorFound);
         throw;
     } catch (...) {
-        fRateLogging = false;
+        ChangeState(fair::mq::Transition::ErrorFound);
         throw;
     }
 
-    // if Run() exited and the state is still RUNNING, transition to READY.
-    if (CheckCurrentState(RUNNING))
-    {
-        ChangeState(internal_READY);
-    }
-
-    PostRun();
-
     rateLogger.get();
 }
 
@@ -409,14 +505,14 @@ void FairMQDevice::HandleSingleChannelInput()
 
     if (fMsgInputs.size() > 0)
     {
-        while (CheckCurrentState(RUNNING) && proceed)
+        while (!NewStatePending() && proceed)
         {
             proceed = HandleMsgInput(fInputChannelKeys.at(0), fMsgInputs.begin()->second, 0);
         }
     }
     else if (fMultipartInputs.size() > 0)
     {
-        while (CheckCurrentState(RUNNING) && proceed)
+        while (!NewStatePending() && proceed)
         {
             proceed = HandleMultipartInput(fInputChannelKeys.at(0), fMultipartInputs.begin()->second, 0);
         }
@@ -468,7 +564,7 @@ void FairMQDevice::HandleMultipleChannelInput()
 
         FairMQPollerPtr poller(fChannels.at(fInputChannelKeys.at(0)).at(0).fTransportFactory->CreatePoller(fChannels, fInputChannelKeys));
 
-        while (CheckCurrentState(RUNNING) && proceed)
+        while (!NewStatePending() && proceed)
         {
             poller->Poll(200);
 
@@ -526,7 +622,7 @@ void FairMQDevice::PollForTransport(const FairMQTransportFactory* factory, const
     {
         FairMQPollerPtr poller(factory->CreatePoller(fChannels, channelKeys));
 
-        while (CheckCurrentState(RUNNING) && fMultitransportProceed)
+        while (!NewStatePending() && fMultitransportProceed)
         {
             poller->Poll(500);
 
@@ -600,58 +696,21 @@ bool FairMQDevice::HandleMultipartInput(const string& chName, const InputMultipa
     }
 }
 
-void FairMQDevice::Run()
+shared_ptr<FairMQTransportFactory> FairMQDevice::AddTransport(fair::mq::Transport transport)
 {
-}
-
-void FairMQDevice::PreRun()
-{
-}
-
-bool FairMQDevice::ConditionalRun()
-{
-    return false;
-}
-
-void FairMQDevice::PostRun()
-{
-}
-
-void FairMQDevice::PauseWrapper()
-{
-    CallStateChangeCallbacks(PAUSED);
-
-    Pause();
-}
-
-void FairMQDevice::Pause()
-{
-    while (CheckCurrentState(PAUSED))
-    {
-        this_thread::sleep_for(chrono::milliseconds(500));
-        LOG(debug) << "paused...";
+    if (transport == fair::mq::Transport::DEFAULT) {
+        transport = fDefaultTransportType;
     }
-    LOG(debug) << "Unpausing";
-}
 
-shared_ptr<FairMQTransportFactory> FairMQDevice::AddTransport(const fair::mq::Transport transport)
-{
     auto i = fTransports.find(transport);
 
-    if (i == fTransports.end())
-    {
+    if (i == fTransports.end()) {
+        LOG(debug) << "Adding '" << fair::mq::TransportNames.at(transport) << "' transport";
         auto tr = FairMQTransportFactory::CreateTransportFactory(fair::mq::TransportNames.at(transport), fId, fConfig);
-
-        LOG(debug) << "Adding '" << fair::mq::TransportNames.at(transport) << "' transport to the device.";
-
-        pair<fair::mq::Transport, shared_ptr<FairMQTransportFactory>> trPair(transport, tr);
-        fTransports.insert(trPair);
-
+        fTransports.insert({transport, tr});
         return tr;
-    }
-    else
-    {
-        LOG(debug) << "Reusing existing '" << fair::mq::TransportNames.at(transport) << "' transport.";
+    } else {
+        LOG(debug) << "Reusing existing '" << fair::mq::TransportNames.at(transport) << "' transport";
         return i->second;
     }
 }
@@ -664,9 +723,6 @@ void FairMQDevice::SetConfig(FairMQProgOptions& config)
 
 void FairMQDevice::LogSocketRates()
 {
-    chrono::time_point<chrono::high_resolution_clock> t0;
-    chrono::time_point<chrono::high_resolution_clock> t1;
-
     vector<FairMQSocket*> filteredSockets;
     vector<string> filteredChannelNames;
     vector<int> logIntervals;
@@ -675,13 +731,10 @@ void FairMQDevice::LogSocketRates()
     size_t chanNameLen = 0;
 
     // iterate over the channels map
-    for (const auto& mi : fChannels)
-    {
+    for (const auto& mi : fChannels) {
         // iterate over the channels vector
-        for (auto vi = (mi.second).begin(); vi != (mi.second).end(); ++vi)
-        {
-            if (vi->fRateLogging > 0)
-            {
+        for (auto vi = (mi.second).begin(); vi != (mi.second).end(); ++vi) {
+            if (vi->fRateLogging > 0) {
                 filteredSockets.push_back(vi->fSocket.get());
                 logIntervals.push_back(vi->fRateLogging);
                 intervalCounters.push_back(0);
@@ -691,55 +744,50 @@ void FairMQDevice::LogSocketRates()
         }
     }
 
-    unsigned int numFilteredSockets = filteredSockets.size();
+    vector<unsigned long> bytesIn(filteredSockets.size());
+    vector<unsigned long> msgIn(filteredSockets.size());
+    vector<unsigned long> bytesOut(filteredSockets.size());
+    vector<unsigned long> msgOut(filteredSockets.size());
 
-    if (numFilteredSockets > 0)
-    {
-        vector<unsigned long> bytesIn(numFilteredSockets);
-        vector<unsigned long> msgIn(numFilteredSockets);
-        vector<unsigned long> bytesOut(numFilteredSockets);
-        vector<unsigned long> msgOut(numFilteredSockets);
+    vector<unsigned long> bytesInNew(filteredSockets.size());
+    vector<unsigned long> msgInNew(filteredSockets.size());
+    vector<unsigned long> bytesOutNew(filteredSockets.size());
+    vector<unsigned long> msgOutNew(filteredSockets.size());
 
-        vector<unsigned long> bytesInNew(numFilteredSockets);
-        vector<unsigned long> msgInNew(numFilteredSockets);
-        vector<unsigned long> bytesOutNew(numFilteredSockets);
-        vector<unsigned long> msgOutNew(numFilteredSockets);
+    vector<double> mbPerSecIn(filteredSockets.size());
+    vector<double> msgPerSecIn(filteredSockets.size());
+    vector<double> mbPerSecOut(filteredSockets.size());
+    vector<double> msgPerSecOut(filteredSockets.size());
 
-        vector<double> mbPerSecIn(numFilteredSockets);
-        vector<double> msgPerSecIn(numFilteredSockets);
-        vector<double> mbPerSecOut(numFilteredSockets);
-        vector<double> msgPerSecOut(numFilteredSockets);
+    int i = 0;
+    for (const auto& vi : filteredSockets) {
+        bytesIn.at(i) = vi->GetBytesRx();
+        bytesOut.at(i) = vi->GetBytesTx();
+        msgIn.at(i) = vi->GetMessagesRx();
+        msgOut.at(i) = vi->GetMessagesTx();
+        ++i;
+    }
 
-        int i = 0;
-        for (const auto& vi : filteredSockets)
-        {
-            bytesIn.at(i) = vi->GetBytesRx();
-            bytesOut.at(i) = vi->GetBytesTx();
-            msgIn.at(i) = vi->GetMessagesRx();
-            msgOut.at(i) = vi->GetMessagesTx();
-            ++i;
-        }
+    chrono::time_point<chrono::high_resolution_clock> t0(chrono::high_resolution_clock::now());
+    chrono::time_point<chrono::high_resolution_clock> t1;
+    uint64_t secondsElapsed = 0;
 
-        t0 = chrono::high_resolution_clock::now();
+    while (!NewStatePending()) {
+        WaitFor(chrono::seconds(1));
 
-        LOG(debug) << "<channel>: in: <#msgs> (<MB>) out: <#msgs> (<MB>)";
+        t1 = chrono::high_resolution_clock::now();
 
-        while (fRateLogging)
-        {
-            t1 = chrono::high_resolution_clock::now();
+        uint64_t msSinceLastLog = chrono::duration_cast<chrono::milliseconds>(t1 - t0).count();
 
-            unsigned long long msSinceLastLog = chrono::duration_cast<chrono::milliseconds>(t1 - t0).count();
+        i = 0;
 
-            i = 0;
+        for (const auto& vi : filteredSockets) {
+            intervalCounters.at(i)++;
 
-            for (const auto& vi : filteredSockets)
-            {
-                intervalCounters.at(i)++;
-
-                if (intervalCounters.at(i) == logIntervals.at(i))
-                {
-                    intervalCounters.at(i) = 0;
+            if (intervalCounters.at(i) == logIntervals.at(i)) {
+                intervalCounters.at(i) = 0;
 
+                if (msSinceLastLog > 0) {
                     bytesInNew.at(i) = vi->GetBytesRx();
                     msgInNew.at(i) = vi->GetMessagesRx();
                     bytesOutNew.at(i) = vi->GetBytesTx();
@@ -759,58 +807,42 @@ void FairMQDevice::LogSocketRates()
                               << "in: " << msgPerSecIn.at(i) << " (" << mbPerSecIn.at(i) << " MB) "
                               << "out: " << msgPerSecOut.at(i) << " (" << mbPerSecOut.at(i) << " MB)";
                 }
-
-                ++i;
             }
 
-            t0 = t1;
-            WaitFor(chrono::milliseconds(1000));
+            ++i;
+        }
+
+        t0 = t1;
+        if (fMaxRunRuntimeInS > 0 && ++secondsElapsed >= fMaxRunRuntimeInS) {
+            ChangeState(fair::mq::Transition::Stop);
         }
     }
 }
 
-void FairMQDevice::Unblock()
+void FairMQDevice::UnblockTransports()
 {
-    for (auto& t : fTransports)
-    {
+    for (auto& t : fTransports) {
         t.second->Interrupt();
     }
-    {
-        lock_guard<mutex> guard(fInterruptedMtx);
-        fInterrupted = true;
-        fRateLogging = false;
-    }
-    fInterruptedCV.notify_all();
 }
 
 void FairMQDevice::ResetTaskWrapper()
 {
-    CallStateChangeCallbacks(RESETTING_TASK);
-
     ResetTask();
 
-    ChangeState(internal_DEVICE_READY);
-}
-
-void FairMQDevice::ResetTask()
-{
+    ChangeState(fair::mq::Transition::Auto);
 }
 
 void FairMQDevice::ResetWrapper()
 {
-    CallStateChangeCallbacks(RESETTING_DEVICE);
-
-    for (auto& t : fTransports)
-    {
+    for (auto& t : fTransports) {
         t.second->Reset();
     }
 
     // iterate over the channels map
-    for (auto& mi : fChannels)
-    {
+    for (auto& mi : fChannels) {
         // iterate over the channels vector
-        for (auto& vi : mi.second)
-        {
+        for (auto& vi : mi.second) {
             // vi.fReset = true;
             vi.fSocket.reset(); // destroy FairMQSocket
         }
@@ -818,18 +850,12 @@ void FairMQDevice::ResetWrapper()
 
     Reset();
 
-    ChangeState(internal_IDLE);
-}
-
-void FairMQDevice::Reset()
-{
-}
-
-void FairMQDevice::Exit()
-{
+    ChangeState(fair::mq::Transition::Auto);
 }
 
 FairMQDevice::~FairMQDevice()
 {
-    LOG(debug) << "Destructing device " << fId;
+    UnsubscribeFromNewTransition("device");
+    fStateMachine.StopHandlingStates();
+    LOG(debug) << "Shutting down device " << fId;
 }

fairmq/FairMQDevice.h

@@ -9,7 +9,7 @@
 #ifndef FAIRMQDEVICE_H_
 #define FAIRMQDEVICE_H_
 
-#include <FairMQStateMachine.h>
+#include <StateMachine.h>
 #include <FairMQTransportFactory.h>
 #include <fairmq/Transports.h>
 
@@ -32,6 +32,7 @@
 #include <assert.h> // static_assert
 #include <type_traits> // is_trivially_copyable
 #include <stdexcept>
+#include <queue>
 
 #include <mutex>
 #include <condition_variable>
@@ -43,11 +44,43 @@ using FairMQChannelMap = std::unordered_map<std::string, std::vector<FairMQChann
 using InputMsgCallback = std::function<bool(FairMQMessagePtr&, int)>;
 using InputMultipartCallback = std::function<bool(FairMQParts&, int)>;
 
-class FairMQDevice : public FairMQStateMachine
+class FairMQDevice
 {
     friend class FairMQChannel;
 
   public:
+    // backwards-compatibility enum for old state machine interface, todo: delete this
+    enum Event
+    {
+        INIT_DEVICE,
+        internal_DEVICE_READY,
+        INIT_TASK,
+        internal_READY,
+        RUN,
+        STOP,
+        RESET_TASK,
+        RESET_DEVICE,
+        internal_IDLE,
+        END,
+        ERROR_FOUND
+    };
+
+    // backwards-compatibility enum for old state machine interface, todo: delete this
+    enum State
+    {
+        OK,
+        Error,
+        IDLE,
+        INITIALIZING_DEVICE,
+        DEVICE_READY,
+        INITIALIZING_TASK,
+        READY,
+        RUNNING,
+        RESETTING_TASK,
+        RESETTING_DEVICE,
+        EXITING
+    };
+
     /// Default constructor
     FairMQDevice();
     /// Constructor with external FairMQProgOptions
@@ -70,9 +103,6 @@ class FairMQDevice : public FairMQStateMachine
     /// Default destructor
     virtual ~FairMQDevice();
 
-    /// Catches interrupt signals (SIGINT, SIGTERM)
-    void CatchSignals();
-
     /// Outputs the socket transfer rates
     virtual void LogSocketRates();
 
@@ -197,9 +227,9 @@ class FairMQDevice : public FairMQStateMachine
         return GetChannel(channel, index).NewSimpleMessage(data);
     }
 
-    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size)
+    FairMQUnmanagedRegionPtr NewUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr)
     {
-        return Transport()->CreateUnmanagedRegion(size);
+        return Transport()->CreateUnmanagedRegion(size, callback);
     }
 
     FairMQUnmanagedRegionPtr NewUnmanagedRegionFor(const std::string& channel, int index, const size_t size, FairMQRegionCallback callback = nullptr)
@@ -330,7 +360,6 @@ class FairMQDevice : public FairMQStateMachine
     } catch (const std::out_of_range& oor) {
         LOG(error) << "out of range: " << oor.what();
         LOG(error) << "requested channel has not been configured? check channel names/configuration.";
-        fRateLogging = false;
         throw;
     }
 
@@ -339,8 +368,7 @@ class FairMQDevice : public FairMQStateMachine
     bool RegisterChannelEndpoint(const std::string& channelName, uint16_t minNumSubChannels = 1, uint16_t maxNumSubChannels = 1)
     {
         bool ok = fChannelRegistry.insert(std::make_pair(channelName, std::make_pair(minNumSubChannels, maxNumSubChannels))).second;
-        if (!ok)
-        {
+        if (!ok) {
             LOG(warn) << "Registering channel: name already registered: \"" << channelName << "\"";
         }
         return ok;
@@ -348,14 +376,10 @@ class FairMQDevice : public FairMQStateMachine
 
     void PrintRegisteredChannels()
     {
-        if (fChannelRegistry.size() < 1)
-        {
+        if (fChannelRegistry.size() < 1) {
             std::cout << "no channels registered." << std::endl;
-        }
-        else
-        {
-            for (const auto& c : fChannelRegistry)
-            {
+        } else {
+            for (const auto& c : fChannelRegistry) {
                 std::cout << c.first << ":" << c.second.first << ":" << c.second.second << std::endl;
             }
         }
@@ -389,8 +413,7 @@ class FairMQDevice : public FairMQStateMachine
 
     void RunStateMachine()
     {
-      CallStateChangeCallbacks(FairMQStateMachine::IDLE);
-      ProcessWork();
+        fStateMachine.ProcessWork();
     };
 
     /// Wait for the supplied amount of time or for interruption.
@@ -399,8 +422,7 @@ class FairMQDevice : public FairMQStateMachine
     template<class Rep, class Period>
     bool WaitFor(std::chrono::duration<Rep, Period> const& duration)
     {
-        std::unique_lock<std::mutex> lock(fInterruptedMtx);
-        return !fInterruptedCV.wait_for(lock, duration, [&] { return fInterrupted.load(); }); // return true if no interruption happened
+        return fStateMachine.WaitForPendingStateFor(std::chrono::duration_cast<std::chrono::milliseconds>(duration).count());
     }
 
   protected:
@@ -421,53 +443,92 @@ class FairMQDevice : public FairMQStateMachine
     std::string fId; ///< Device ID
 
     /// Additional user initialization (can be overloaded in child classes). Prefer to use InitTask().
-    virtual void Init();
+    virtual void Init() {}
+
+    virtual void Bind() {}
+
+    virtual void Connect() {}
 
     /// Task initialization (can be overloaded in child classes)
-    virtual void InitTask();
+    virtual void InitTask() {}
 
     /// Runs the device (to be overloaded in child classes)
-    virtual void Run();
+    virtual void Run() {}
 
     /// Called in the RUNNING state once before executing the Run()/ConditionalRun() method
-    virtual void PreRun();
+    virtual void PreRun() {}
 
     /// Called during RUNNING state repeatedly until it returns false or device state changes
-    virtual bool ConditionalRun();
+    virtual bool ConditionalRun() { return false; }
 
     /// Called in the RUNNING state once after executing the Run()/ConditionalRun() method
-    virtual void PostRun();
+    virtual void PostRun() {}
 
-    /// Handles the PAUSE state
-    virtual void Pause();
+    virtual void Pause() __attribute__((deprecated("PAUSE state is removed. This method is never called. To pause Run, go to READY with STOP transition and back to RUNNING with RUN to resume."))) {}
 
     /// Resets the user task (to be overloaded in child classes)
-    virtual void ResetTask();
+    virtual void ResetTask() {}
 
     /// Resets the device (can be overloaded in child classes)
-    virtual void Reset();
+    virtual void Reset() {}
+
+  public:
+    bool ChangeState(const fair::mq::Transition transition) { return fStateMachine.ChangeState(transition); }
+    bool ChangeState(const std::string& transition) { return fStateMachine.ChangeState(fair::mq::StateMachine::GetTransition(transition)); }
+
+    bool ChangeState(const int transition) __attribute__((deprecated("Use ChangeState(const fair::mq::Transition transition).")));
+
+    void WaitForEndOfState(const fair::mq::Transition transition) __attribute__((deprecated("Use WaitForState(fair::mq::State expectedState).")));
+    void WaitForEndOfState(const std::string& transition) __attribute__((deprecated("Use WaitForState(fair::mq::State expectedState)."))) { WaitForState(transition); }
+
+    fair::mq::State WaitForNextState();
+    void WaitForState(fair::mq::State state);
+    void WaitForState(const std::string& state) { fair::mq::StateMachine::GetState(state); }
+
+    void SubscribeToStateChange(const std::string& key, std::function<void(const fair::mq::State)> callback) { fStateMachine.SubscribeToStateChange(key, callback); }
+    void UnsubscribeFromStateChange(const std::string& key) { fStateMachine.UnsubscribeFromStateChange(key); }
+
+    void SubscribeToNewTransition(const std::string& key, std::function<void(const fair::mq::Transition)> callback) { fStateMachine.SubscribeToNewTransition(key, callback); }
+    void UnsubscribeFromNewTransition(const std::string& key) { fStateMachine.UnsubscribeFromNewTransition(key); }
+
+    bool CheckCurrentState(const int /* state */) const __attribute__((deprecated("Use NewStatePending()."))) { return !fStateMachine.NewStatePending(); }
+    bool CheckCurrentState(const std::string& /* state */) const __attribute__((deprecated("Use NewStatePending()."))) { return !fStateMachine.NewStatePending(); }
+
+    /// Returns true is a new state has been requested, signaling the current handler to stop.
+    bool NewStatePending() const { return fStateMachine.NewStatePending(); }
+
+    fair::mq::State GetCurrentState() const { return fStateMachine.GetCurrentState(); }
+    std::string GetCurrentStateName() const { return fStateMachine.GetCurrentStateName(); }
+
+    static std::string GetStateName(const fair::mq::State state) { return fair::mq::StateMachine::GetStateName(state); }
+    static std::string GetTransitionName(const fair::mq::Transition transition) { return fair::mq::StateMachine::GetTransitionName(transition); }
+
+    struct DeviceStateError : std::runtime_error { using std::runtime_error::runtime_error; };
 
   private:
     fair::mq::Transport fDefaultTransportType; ///< Default transport for the device
+    fair::mq::StateMachine fStateMachine;
 
-    /// Handles the initialization and the Init() method
+    /// Handles the initialization
     void InitWrapper();
+    /// Initializes binding channels
+    void BindWrapper();
+    /// Initializes connecting channels
+    void ConnectWrapper();
     /// Handles the InitTask() method
     void InitTaskWrapper();
     /// Handles the Run() method
     void RunWrapper();
-    /// Handles the Pause() method
-    void PauseWrapper();
     /// Handles the ResetTask() method
     void ResetTaskWrapper();
     /// Handles the Reset() method
     void ResetWrapper();
 
-    /// Unblocks blocking channel send/receive calls
-    void Unblock();
+    /// Notifies transports to cease any blocking activity
+    void UnblockTransports();
 
     /// Shuts down the transports and the device
-    void Exit();
+    void Exit() {}
 
     /// Attach (bind/connect) channels in the list
     void AttachChannels(std::vector<FairMQChannel*>& chans);
@@ -481,7 +542,8 @@ class FairMQDevice : public FairMQStateMachine
     bool HandleMsgInput(const std::string& chName, const InputMsgCallback& callback, int i);
     bool HandleMultipartInput(const std::string& chName, const InputMultipartCallback& callback, int i);
 
-    void CreateOwnConfig();
+    std::vector<FairMQChannel*> fUninitializedBindingChannels;
+    std::vector<FairMQChannel*> fUninitializedConnectingChannels;
 
     bool fDataCallbacks;
     std::unordered_map<std::string, InputMsgCallback> fMsgInputs;
@@ -494,12 +556,12 @@ class FairMQDevice : public FairMQStateMachine
 
     const fair::mq::tools::Version fVersion;
     float fRate; ///< Rate limiting for ConditionalRun
+    uint64_t fMaxRunRuntimeInS; ///< Maximum runtime for the Running state handler, after which state will change to Ready (in seconds, 0 for no limit).
     std::vector<std::string> fRawCmdLineArgs;
 
-    std::atomic<bool> fInterrupted;
-    std::condition_variable fInterruptedCV;
-    std::mutex fInterruptedMtx;
-    mutable std::atomic<bool> fRateLogging;
+    std::queue<fair::mq::State> fStates;
+    std::mutex fStatesMtx;
+    std::condition_variable fStatesCV;
 };
 
 #endif /* FAIRMQDEVICE_H_ */

fairmq/FairMQMessage.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/FairMQPoller.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/FairMQPoller.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/FairMQSocket.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/FairMQSocket.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/FairMQStateMachine.cxx

@@ -1,672 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQStateMachine.cxx
- *
- * @since 2012-10-25
- * @author D. Klein, A. Rybalchenko
- */
-
-#include "FairMQStateMachine.h"
-#include <fairmq/Tools.h>
-
-// Increase maximum number of boost::msm states (default is 10)
-// This #define has to be before any msm header includes
-#define FUSION_MAX_VECTOR_SIZE 20
-
-#include <boost/mpl/for_each.hpp>
-#include <boost/msm/back/state_machine.hpp>
-#include <boost/msm/back/tools.hpp>
-#include <boost/msm/back/metafunctions.hpp>
-#include <boost/msm/front/state_machine_def.hpp>
-#include <boost/msm/front/functor_row.hpp>
-#include <boost/core/demangle.hpp>
-#include <boost/signals2.hpp> // signal/slot for onStateChange callbacks
-
-#include <atomic>
-#include <condition_variable>
-#include <chrono>
-#include <array>
-#include <unordered_map>
-
-using namespace std;
-using namespace boost::msm::front;
-
-namespace std
-{
-
-template<>
-struct hash<FairMQStateMachine::Event> : fair::mq::tools::HashEnum<FairMQStateMachine::Event> {};
-
-} /* namespace std */
-
-namespace fair
-{
-namespace mq
-{
-namespace fsm
-{
-
-// list of FSM states
-struct OK_FSM_STATE                  : public state<> { static string Name() { return "OK"; }                  static FairMQStateMachine::State Type() { return FairMQStateMachine::State::OK; } };
-struct ERROR_FSM_STATE               : public terminate_state<> { static string Name() { return "ERROR"; }     static FairMQStateMachine::State Type() { return FairMQStateMachine::State::Error; } };
-
-struct IDLE_FSM_STATE                : public state<> { static string Name() { return "IDLE"; }                static FairMQStateMachine::State Type() { return FairMQStateMachine::State::IDLE; } };
-struct INITIALIZING_DEVICE_FSM_STATE : public state<> { static string Name() { return "INITIALIZING_DEVICE"; } static FairMQStateMachine::State Type() { return FairMQStateMachine::State::INITIALIZING_DEVICE; } };
-struct DEVICE_READY_FSM_STATE        : public state<> { static string Name() { return "DEVICE_READY"; }        static FairMQStateMachine::State Type() { return FairMQStateMachine::State::DEVICE_READY; } };
-struct INITIALIZING_TASK_FSM_STATE   : public state<> { static string Name() { return "INITIALIZING_TASK"; }   static FairMQStateMachine::State Type() { return FairMQStateMachine::State::INITIALIZING_TASK; } };
-struct READY_FSM_STATE               : public state<> { static string Name() { return "READY"; }               static FairMQStateMachine::State Type() { return FairMQStateMachine::State::READY; } };
-struct RUNNING_FSM_STATE             : public state<> { static string Name() { return "RUNNING"; }             static FairMQStateMachine::State Type() { return FairMQStateMachine::State::RUNNING; } };
-struct PAUSED_FSM_STATE              : public state<> { static string Name() { return "PAUSED"; }              static FairMQStateMachine::State Type() { return FairMQStateMachine::State::PAUSED; } };
-struct RESETTING_TASK_FSM_STATE      : public state<> { static string Name() { return "RESETTING_TASK"; }      static FairMQStateMachine::State Type() { return FairMQStateMachine::State::RESETTING_TASK; } };
-struct RESETTING_DEVICE_FSM_STATE    : public state<> { static string Name() { return "RESETTING_DEVICE"; }    static FairMQStateMachine::State Type() { return FairMQStateMachine::State::RESETTING_DEVICE; } };
-struct EXITING_FSM_STATE             : public state<> { static string Name() { return "EXITING"; }             static FairMQStateMachine::State Type() { return FairMQStateMachine::State::EXITING; } };
-
-// list of FSM events
-struct INIT_DEVICE_FSM_EVENT           { static string Name() { return "INIT_DEVICE"; }           static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::INIT_DEVICE; } };
-struct internal_DEVICE_READY_FSM_EVENT { static string Name() { return "internal_DEVICE_READY"; } static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::internal_DEVICE_READY; } };
-struct INIT_TASK_FSM_EVENT             { static string Name() { return "INIT_TASK"; }             static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::INIT_TASK; } };
-struct internal_READY_FSM_EVENT        { static string Name() { return "internal_READY"; }        static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::internal_READY; } };
-struct RUN_FSM_EVENT                   { static string Name() { return "RUN"; }                   static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::RUN; } };
-struct PAUSE_FSM_EVENT                 { static string Name() { return "PAUSE"; }                 static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::PAUSE; } };
-struct STOP_FSM_EVENT                  { static string Name() { return "STOP"; }                  static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::STOP; } };
-struct RESET_TASK_FSM_EVENT            { static string Name() { return "RESET_TASK"; }            static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::RESET_TASK; } };
-struct RESET_DEVICE_FSM_EVENT          { static string Name() { return "RESET_DEVICE"; }          static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::RESET_DEVICE; } };
-struct internal_IDLE_FSM_EVENT         { static string Name() { return "internal_IDLE"; }         static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::internal_IDLE; } };
-struct END_FSM_EVENT                   { static string Name() { return "END"; }                   static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::END; } };
-struct ERROR_FOUND_FSM_EVENT           { static string Name() { return "ERROR_FOUND"; }           static FairMQStateMachine::Event Type() { return FairMQStateMachine::Event::ERROR_FOUND; } };
-
-static array<string, 12> stateNames =
-{
-    {
-        "OK",
-        "Error",
-        "IDLE",
-        "INITIALIZING_DEVICE",
-        "DEVICE_READY",
-        "INITIALIZING_TASK",
-        "READY",
-        "RUNNING",
-        "PAUSED",
-        "RESETTING_TASK",
-        "RESETTING_DEVICE",
-        "EXITING"
-    }
-};
-
-static array<string, 12> eventNames =
-{
-    {
-        "INIT_DEVICE",
-        "internal_DEVICE_READY",
-        "INIT_TASK",
-        "internal_READY",
-        "RUN",
-        "PAUSE",
-        "STOP",
-        "RESET_TASK",
-        "RESET_DEVICE",
-        "internal_IDLE",
-        "END",
-        "ERROR_FOUND"
-    }
-};
-
-static map<string, int> stateNumbers =
-{
-    { "OK",                  FairMQStateMachine::State::OK },
-    { "Error",               FairMQStateMachine::State::Error },
-    { "IDLE",                FairMQStateMachine::State::IDLE },
-    { "INITIALIZING_DEVICE", FairMQStateMachine::State::INITIALIZING_DEVICE },
-    { "DEVICE_READY",        FairMQStateMachine::State::DEVICE_READY },
-    { "INITIALIZING_TASK",   FairMQStateMachine::State::INITIALIZING_TASK },
-    { "READY",               FairMQStateMachine::State::READY },
-    { "RUNNING",             FairMQStateMachine::State::RUNNING },
-    { "PAUSED",              FairMQStateMachine::State::PAUSED },
-    { "RESETTING_TASK",      FairMQStateMachine::State::RESETTING_TASK },
-    { "RESETTING_DEVICE",    FairMQStateMachine::State::RESETTING_DEVICE },
-    { "EXITING",             FairMQStateMachine::State::EXITING }
-};
-
-static map<string, int> eventNumbers =
-{
-    { "INIT_DEVICE",           FairMQStateMachine::Event::INIT_DEVICE },
-    { "internal_DEVICE_READY", FairMQStateMachine::Event::internal_DEVICE_READY },
-    { "INIT_TASK",             FairMQStateMachine::Event::INIT_TASK },
-    { "internal_READY",        FairMQStateMachine::Event::internal_READY },
-    { "RUN",                   FairMQStateMachine::Event::RUN },
-    { "PAUSE",                 FairMQStateMachine::Event::PAUSE },
-    { "STOP",                  FairMQStateMachine::Event::STOP },
-    { "RESET_TASK",            FairMQStateMachine::Event::RESET_TASK },
-    { "RESET_DEVICE",          FairMQStateMachine::Event::RESET_DEVICE },
-    { "internal_IDLE",         FairMQStateMachine::Event::internal_IDLE },
-    { "END",                   FairMQStateMachine::Event::END },
-    { "ERROR_FOUND",           FairMQStateMachine::Event::ERROR_FOUND }
-};
-
-// defining the boost MSM state machine
-struct Machine_ : public state_machine_def<Machine_>
-{
-  public:
-    Machine_()
-        : fUnblockHandler()
-        , fStateHandlers()
-        , fWork()
-        , fWorkAvailableCondition()
-        , fWorkDoneCondition()
-        , fWorkMutex()
-        , fWorkerTerminated(false)
-        , fWorkActive(false)
-        , fWorkAvailable(false)
-        , fStateChangeSignal()
-        , fStateChangeSignalsMap()
-        , fState()
-    {}
-
-    virtual ~Machine_()
-    {}
-
-    // initial states
-    using initial_state = boost::mpl::vector<IDLE_FSM_STATE, OK_FSM_STATE>;
-
-    template<typename Event, typename FSM>
-    void on_entry(Event const&, FSM& /*fsm*/)
-    {
-        LOG(state) << "Starting FairMQ state machine";
-        fState = FairMQStateMachine::IDLE;
-        LOG(state) << "Entering IDLE state";
-        // fsm.CallStateChangeCallbacks(FairMQStateMachine::IDLE);
-        // we call this for now in FairMQDevice::RunStateMachine()
-    }
-
-    template<typename Event, typename FSM>
-    void on_exit(Event const&, FSM& /*fsm*/)
-    {
-        LOG(state) << "Exiting FairMQ state machine";
-    }
-
-    // actions
-    struct AutomaticFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const&, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-            LOG(state) << "Entering " << ts.Name() << " state";
-        }
-    };
-
-    struct DefaultFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const& e, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-
-            unique_lock<mutex> lock(fsm.fWorkMutex);
-            while (fsm.fWorkActive)
-            {
-                fsm.fWorkDoneCondition.wait(lock);
-            }
-            fsm.fWorkAvailable = true;
-            LOG(state) << "Entering " << ts.Name() << " state";
-            fsm.fWork = fsm.fStateHandlers.at(e.Type());
-            fsm.fWorkAvailableCondition.notify_one();
-        }
-    };
-
-    struct PauseFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const&, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-
-            fsm.fUnblockHandler();
-            unique_lock<mutex> lock(fsm.fWorkMutex);
-            while (fsm.fWorkActive)
-            {
-                fsm.fWorkDoneCondition.wait(lock);
-            }
-            fsm.fWorkAvailable = true;
-            LOG(state) << "Entering " << ts.Name() << " state";
-            fsm.fWork = fsm.fPauseWrapperHandler;
-            fsm.fWorkAvailableCondition.notify_one();
-        }
-    };
-
-    struct StopFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const&, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-
-            fsm.fUnblockHandler();
-            unique_lock<mutex> lock(fsm.fWorkMutex);
-            while (fsm.fWorkActive)
-            {
-                fsm.fWorkDoneCondition.wait(lock);
-            }
-            LOG(state) << "Entering " << ts.Name() << " state";
-        }
-    };
-
-    struct InternalStopFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const&, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-            fsm.fUnblockHandler();
-            LOG(state) << "RUNNING state finished without an external event, entering " << ts.Name() << " state";
-        }
-    };
-
-    struct ExitingFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const& e, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            LOG(state) << "Entering " << ts.Name() << " state";
-            fsm.fState = ts.Type();
-            fsm.CallStateChangeCallbacks(FairMQStateMachine::EXITING);
-
-            // Stop ProcessWork()
-            {
-                lock_guard<mutex> lock(fsm.fWorkMutex);
-                fsm.fWorkerTerminated = true;
-                fsm.fWorkAvailableCondition.notify_one();
-            }
-
-            fsm.fStateHandlers.at(e.Type())();
-        }
-    };
-
-    struct ErrorFoundFct
-    {
-        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
-        void operator()(EVT const&, FSM& fsm, SourceState& /* ss */, TargetState& ts)
-        {
-            fsm.fState = ts.Type();
-            LOG(state) << "Entering " << ts.Name() << " state";
-            fsm.CallStateChangeCallbacks(FairMQStateMachine::Error);
-        }
-    };
-
-    // Transition table for Machine_
-    struct transition_table : boost::mpl::vector<
-        // Start                           Event                            Next                           Action           Guard
-        Row<IDLE_FSM_STATE,                INIT_DEVICE_FSM_EVENT,           INITIALIZING_DEVICE_FSM_STATE, DefaultFct,      none>,
-        Row<IDLE_FSM_STATE,                END_FSM_EVENT,                   EXITING_FSM_STATE,             ExitingFct,      none>,
-        Row<INITIALIZING_DEVICE_FSM_STATE, internal_DEVICE_READY_FSM_EVENT, DEVICE_READY_FSM_STATE,        AutomaticFct,    none>,
-        Row<DEVICE_READY_FSM_STATE,        INIT_TASK_FSM_EVENT,             INITIALIZING_TASK_FSM_STATE,   DefaultFct,      none>,
-        Row<DEVICE_READY_FSM_STATE,        RESET_DEVICE_FSM_EVENT,          RESETTING_DEVICE_FSM_STATE,    DefaultFct,      none>,
-        Row<INITIALIZING_TASK_FSM_STATE,   internal_READY_FSM_EVENT,        READY_FSM_STATE,               AutomaticFct,    none>,
-        Row<READY_FSM_STATE,               RUN_FSM_EVENT,                   RUNNING_FSM_STATE,             DefaultFct,      none>,
-        Row<READY_FSM_STATE,               RESET_TASK_FSM_EVENT,            RESETTING_TASK_FSM_STATE,      DefaultFct,      none>,
-        Row<RUNNING_FSM_STATE,             PAUSE_FSM_EVENT,                 PAUSED_FSM_STATE,              DefaultFct,      none>,
-        Row<RUNNING_FSM_STATE,             STOP_FSM_EVENT,                  READY_FSM_STATE,               StopFct,         none>,
-        Row<RUNNING_FSM_STATE,             internal_READY_FSM_EVENT,        READY_FSM_STATE,               InternalStopFct, none>,
-        Row<PAUSED_FSM_STATE,              RUN_FSM_EVENT,                   RUNNING_FSM_STATE,             DefaultFct,      none>,
-        Row<RESETTING_TASK_FSM_STATE,      internal_DEVICE_READY_FSM_EVENT, DEVICE_READY_FSM_STATE,        AutomaticFct,    none>,
-        Row<RESETTING_DEVICE_FSM_STATE,    internal_IDLE_FSM_EVENT,         IDLE_FSM_STATE,                AutomaticFct,    none>,
-        Row<OK_FSM_STATE,                  ERROR_FOUND_FSM_EVENT,           ERROR_FSM_STATE,               ErrorFoundFct,   none>>
-    {};
-
-    // replaces the default no-transition response.
-    template<typename FSM, typename Event>
-    void no_transition(Event const& e, FSM&, int state)
-    {
-        using recursive_stt = typename boost::msm::back::recursive_get_transition_table<FSM>::type;
-        using all_states = typename boost::msm::back::generate_state_set<recursive_stt>::type;
-
-        string stateName;
-
-        boost::mpl::for_each<all_states, boost::msm::wrap<boost::mpl::placeholders::_1>>(boost::msm::back::get_state_name<recursive_stt>(stateName, state));
-
-        stateName = boost::core::demangle(stateName.c_str());
-        size_t pos = stateName.rfind(":");
-        if (pos != string::npos)
-        {
-            stateName = stateName.substr(pos + 1);
-            stateName = stateName.substr(0, stateName.size() - 10);
-        }
-
-        if (stateName != "OK")
-        {
-            LOG(state) << "No transition from state " << stateName << " on event " << e.Name();
-        }
-    }
-
-    void CallStateChangeCallbacks(const FairMQStateMachine::State state) const
-    {
-        if (!fStateChangeSignal.empty())
-        {
-            fStateChangeSignal(state);
-        }
-    }
-
-    function<void(void)> fUnblockHandler;
-    unordered_map<FairMQStateMachine::Event, function<void(void)>> fStateHandlers;
-
-    // function to execute user states in a worker thread
-    function<void(void)> fWork;
-    condition_variable fWorkAvailableCondition;
-    condition_variable fWorkDoneCondition;
-    mutex fWorkMutex;
-    bool fWorkerTerminated;
-    bool fWorkActive;
-    bool fWorkAvailable;
-
-    boost::signals2::signal<void(const FairMQStateMachine::State)> fStateChangeSignal;
-    unordered_map<string, boost::signals2::connection> fStateChangeSignalsMap;
-
-    atomic<FairMQStateMachine::State> fState;
-
-    void ProcessWork()
-    {
-        while (true)
-        {
-            {
-                unique_lock<mutex> lock(fWorkMutex);
-                // Wait for work to be done.
-                while (!fWorkAvailable && !fWorkerTerminated)
-                {
-                    fWorkAvailableCondition.wait_for(lock, chrono::milliseconds(100));
-                }
-
-                if (fWorkerTerminated)
-                {
-                    break;
-                }
-
-                fWorkActive = true;
-            }
-
-            fWork();
-
-            {
-                lock_guard<mutex> lock(fWorkMutex);
-                fWorkActive = false;
-                fWorkAvailable = false;
-                fWorkDoneCondition.notify_one();
-            }
-            CallStateChangeCallbacks(fState);
-        }
-    }
-}; // Machine_
-
-using FairMQFSM = boost::msm::back::state_machine<Machine_>;
-
-} // namespace fsm
-} // namespace mq
-} // namespace fair
-
-using namespace fair::mq::fsm;
-
-FairMQStateMachine::FairMQStateMachine()
-    : fChangeStateMutex()
-    , fFsm(new FairMQFSM)
-{
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(INIT_DEVICE, bind(&FairMQStateMachine::InitWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(INIT_TASK, bind(&FairMQStateMachine::InitTaskWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(RUN, bind(&FairMQStateMachine::RunWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(PAUSE, bind(&FairMQStateMachine::PauseWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(RESET_TASK, bind(&FairMQStateMachine::ResetTaskWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(RESET_DEVICE, bind(&FairMQStateMachine::ResetWrapper, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateHandlers.emplace(END, bind(&FairMQStateMachine::Exit, this));
-    static_pointer_cast<FairMQFSM>(fFsm)->fUnblockHandler = bind(&FairMQStateMachine::Unblock, this);
-
-    static_pointer_cast<FairMQFSM>(fFsm)->start();
-}
-
-FairMQStateMachine::~FairMQStateMachine()
-{
-    static_pointer_cast<FairMQFSM>(fFsm)->stop();
-}
-
-int FairMQStateMachine::GetInterfaceVersion() const
-{
-    return FAIRMQ_INTERFACE_VERSION;
-}
-
-bool FairMQStateMachine::ChangeState(int event)
-{
-    try
-    {
-        switch (event)
-        {
-            case INIT_DEVICE:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(INIT_DEVICE_FSM_EVENT());
-                return true;
-            }
-            case internal_DEVICE_READY:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(internal_DEVICE_READY_FSM_EVENT());
-                return true;
-            }
-            case INIT_TASK:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(INIT_TASK_FSM_EVENT());
-                return true;
-            }
-            case internal_READY:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(internal_READY_FSM_EVENT());
-                return true;
-            }
-            case RUN:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(RUN_FSM_EVENT());
-                return true;
-            }
-            case PAUSE:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(PAUSE_FSM_EVENT());
-                return true;
-            }
-            case STOP:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(STOP_FSM_EVENT());
-                return true;
-            }
-            case RESET_DEVICE:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(RESET_DEVICE_FSM_EVENT());
-                return true;
-            }
-            case RESET_TASK:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(RESET_TASK_FSM_EVENT());
-                return true;
-            }
-            case internal_IDLE:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(internal_IDLE_FSM_EVENT());
-                return true;
-            }
-            case END:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(END_FSM_EVENT());
-                return true;
-            }
-            case ERROR_FOUND:
-            {
-                lock_guard<mutex> lock(fChangeStateMutex);
-                static_pointer_cast<FairMQFSM>(fFsm)->process_event(ERROR_FOUND_FSM_EVENT());
-                return true;
-            }
-            default:
-            {
-                LOG(error) << "Requested state transition with an unsupported event: " << event << endl
-                           << "Supported are: INIT_DEVICE, INIT_TASK, RUN, PAUSE, STOP, RESET_TASK, RESET_DEVICE, END, ERROR_FOUND";
-                return false;
-            }
-        }
-    }
-    catch (exception& e)
-    {
-        LOG(error) << "Exception in FairMQStateMachine::ChangeState(): " << e.what();
-        exit(EXIT_FAILURE);
-    }
-    return false;
-}
-
-bool FairMQStateMachine::ChangeState(const string& event)
-{
-    return ChangeState(GetEventNumber(event));
-}
-
-void FairMQStateMachine::WaitForEndOfState(int event)
-{
-    try
-    {
-        switch (event)
-        {
-            case INIT_DEVICE:
-            case INIT_TASK:
-            case RUN:
-            case RESET_TASK:
-            case RESET_DEVICE:
-            {
-                unique_lock<mutex> lock(static_pointer_cast<FairMQFSM>(fFsm)->fWorkMutex);
-                while (static_pointer_cast<FairMQFSM>(fFsm)->fWorkActive || static_pointer_cast<FairMQFSM>(fFsm)->fWorkAvailable)
-                {
-                    static_pointer_cast<FairMQFSM>(fFsm)->fWorkDoneCondition.wait_for(lock, chrono::seconds(1));
-                }
-
-                break;
-            }
-            default:
-                LOG(error) << "Requested state is either synchronous or does not exist.";
-                break;
-        }
-    }
-    catch (exception& e)
-    {
-        LOG(error) << "Exception in FairMQStateMachine::WaitForEndOfState(): " << e.what();
-    }
-}
-
-void FairMQStateMachine::WaitForEndOfState(const string& event)
-{
-    return WaitForEndOfState(GetEventNumber(event));
-}
-
-bool FairMQStateMachine::WaitForEndOfStateForMs(int event, int durationInMs)
-{
-    try
-    {
-        switch (event)
-        {
-            case INIT_DEVICE:
-            case INIT_TASK:
-            case RUN:
-            case RESET_TASK:
-            case RESET_DEVICE:
-            {
-                unique_lock<mutex> lock(static_pointer_cast<FairMQFSM>(fFsm)->fWorkMutex);
-                while (static_pointer_cast<FairMQFSM>(fFsm)->fWorkActive || static_pointer_cast<FairMQFSM>(fFsm)->fWorkAvailable)
-                {
-                    static_pointer_cast<FairMQFSM>(fFsm)->fWorkDoneCondition.wait_for(lock, chrono::milliseconds(durationInMs));
-                    if (static_pointer_cast<FairMQFSM>(fFsm)->fWorkActive)
-                    {
-                        return false;
-                    }
-                }
-                return true;
-            }
-            default:
-                LOG(error) << "Requested state is either synchronous or does not exist.";
-                return false;
-        }
-    }
-    catch (exception& e)
-    {
-        LOG(error) << "Exception in FairMQStateMachine::WaitForEndOfStateForMs(): " << e.what();
-    }
-    return false;
-}
-
-bool FairMQStateMachine::WaitForEndOfStateForMs(const string& event, int durationInMs)
-{
-    return WaitForEndOfStateForMs(GetEventNumber(event), durationInMs);
-}
-
-void FairMQStateMachine::SubscribeToStateChange(const string& key, function<void(const State)> callback)
-{
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignalsMap.insert({key, static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignal.connect(callback)});
-}
-void FairMQStateMachine::UnsubscribeFromStateChange(const string& key)
-{
-    if (static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignalsMap.count(key))
-    {
-        static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignalsMap.at(key).disconnect();
-        static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignalsMap.erase(key);
-    }
-}
-
-void FairMQStateMachine::CallStateChangeCallbacks(const State state) const
-{
-    static_pointer_cast<FairMQFSM>(fFsm)->CallStateChangeCallbacks(state);
-}
-
-string FairMQStateMachine::GetCurrentStateName() const
-{
-    return GetStateName(static_pointer_cast<FairMQFSM>(fFsm)->fState);
-}
-string FairMQStateMachine::GetStateName(const State state)
-{
-    return stateNames.at(state);
-}
-int FairMQStateMachine::GetCurrentState() const
-{
-    return static_pointer_cast<FairMQFSM>(fFsm)->fState;
-}
-bool FairMQStateMachine::CheckCurrentState(int state) const
-{
-    return state == static_pointer_cast<FairMQFSM>(fFsm)->fState;
-}
-bool FairMQStateMachine::CheckCurrentState(const string& state) const
-{
-    return state == GetCurrentStateName();
-}
-
-void FairMQStateMachine::ProcessWork()
-try
-{
-    static_pointer_cast<FairMQFSM>(fFsm)->ProcessWork();
-} catch(...) {
-    {
-        lock_guard<mutex> lock(static_pointer_cast<FairMQFSM>(fFsm)->fWorkMutex);
-        static_pointer_cast<FairMQFSM>(fFsm)->fWorkActive = false;
-        static_pointer_cast<FairMQFSM>(fFsm)->fWorkAvailable = false;
-        static_pointer_cast<FairMQFSM>(fFsm)->fWorkDoneCondition.notify_one();
-    }
-    ChangeState(ERROR_FOUND);
-    throw;
-}
-
-
-int FairMQStateMachine::GetEventNumber(const string& event)
-{
-    return eventNumbers.at(event);
-}

fairmq/FairMQStateMachine.h

@@ -1,107 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-/**
- * FairMQStateMachine.h
- *
- * @since 2012-10-25
- * @author D. Klein, A. Rybalchenko
- */
-
-#ifndef FAIRMQSTATEMACHINE_H_
-#define FAIRMQSTATEMACHINE_H_
-
-#define FAIRMQ_INTERFACE_VERSION 3
-
-#include "FairMQLogger.h"
-
-#include <string>
-#include <memory>
-#include <functional>
-#include <mutex>
-
-class FairMQStateMachine
-{
-  public:
-    enum Event
-    {
-        INIT_DEVICE,
-        internal_DEVICE_READY,
-        INIT_TASK,
-        internal_READY,
-        RUN,
-        PAUSE,
-        STOP,
-        RESET_TASK,
-        RESET_DEVICE,
-        internal_IDLE,
-        END,
-        ERROR_FOUND
-    };
-
-    enum State
-    {
-        OK,
-        Error,
-        IDLE,
-        INITIALIZING_DEVICE,
-        DEVICE_READY,
-        INITIALIZING_TASK,
-        READY,
-        RUNNING,
-        PAUSED,
-        RESETTING_TASK,
-        RESETTING_DEVICE,
-        EXITING
-    };
-
-    FairMQStateMachine();
-    virtual ~FairMQStateMachine();
-
-    int GetInterfaceVersion() const;
-
-    bool ChangeState(int event);
-    bool ChangeState(const std::string& event);
-
-    void WaitForEndOfState(int event);
-    void WaitForEndOfState(const std::string& event);
-
-    bool WaitForEndOfStateForMs(int event, int durationInMs);
-    bool WaitForEndOfStateForMs(const std::string& event, int durationInMs);
-
-    void SubscribeToStateChange(const std::string& key, std::function<void(const State)> callback);
-    void UnsubscribeFromStateChange(const std::string& key);
-
-    void CallStateChangeCallbacks(const State state) const;
-
-    std::string GetCurrentStateName() const;
-    static std::string GetStateName(const State);
-    int GetCurrentState() const;
-    bool CheckCurrentState(int state) const;
-    bool CheckCurrentState(const std::string& state) const;
-
-    // actions to be overwritten by derived classes
-    virtual void InitWrapper() {}
-    virtual void InitTaskWrapper() {}
-    virtual void RunWrapper() {}
-    virtual void PauseWrapper() {}
-    virtual void ResetWrapper() {}
-    virtual void ResetTaskWrapper() {}
-    virtual void Exit() {}
-    virtual void Unblock() {}
-
-    void ProcessWork();
-
-  private:
-    static int GetEventNumber(const std::string& event);
-
-    std::mutex fChangeStateMutex;
-
-    std::shared_ptr<void> fFsm;
-};
-
-#endif /* FAIRMQSTATEMACHINE_H_ */

fairmq/Plugin.h

@@ -78,7 +78,7 @@ class Plugin
     auto TakeDeviceControl() -> void { fPluginServices->TakeDeviceControl(fkName); };
     auto StealDeviceControl() -> void { fPluginServices->StealDeviceControl(fkName); };
     auto ReleaseDeviceControl() -> void { fPluginServices->ReleaseDeviceControl(fkName); };
-    auto ChangeDeviceState(const DeviceStateTransition next) -> void { fPluginServices->ChangeDeviceState(fkName, next); }
+    auto ChangeDeviceState(const DeviceStateTransition next) -> bool { return fPluginServices->ChangeDeviceState(fkName, next); }
     auto SubscribeToDeviceStateChange(std::function<void(DeviceState)> callback) -> void { fPluginServices->SubscribeToDeviceStateChange(fkName, callback); }
     auto UnsubscribeFromDeviceStateChange() -> void { fPluginServices->UnsubscribeFromDeviceStateChange(fkName); }
 

fairmq/PluginManager.cxx

@@ -29,7 +29,7 @@ using boost::optional;
 const std::string fair::mq::PluginManager::fgkLibPrefix = "FairMQPlugin_";
 
 fair::mq::PluginManager::PluginManager()
-    : fSearchPaths{{"."}}
+    : fSearchPaths{}
     , fPluginFactories()
     , fPluginServices()
     , fPlugins()
@@ -39,7 +39,7 @@ fair::mq::PluginManager::PluginManager()
 }
 
 fair::mq::PluginManager::PluginManager(const vector<string> args)
-    : fSearchPaths{{"."}}
+    : fSearchPaths{}
     , fPluginFactories()
     , fPluginServices()
     , fPlugins()
@@ -115,7 +115,8 @@ auto fair::mq::PluginManager::ProgramOptions() -> po::options_description
                                                                             "* Append(>) or prepend(<) to default search path, e.g.\n"
                                                                             "  -S >/lib </home/user/lib\n"
                                                                             "* If you mix the overriding and appending/prepending syntaxes, the overriding paths act as default search path, e.g.\n"
-                                                                            "  -S /usr/lib >/lib </home/user/lib /usr/local/lib results in /home/user/lib,/usr/local/lib,/usr/lib/,/lib")
+                                                                            "  -S /usr/lib >/lib </home/user/lib /usr/local/lib results in /home/user/lib,/usr/local/lib,/usr/lib/,/lib\n"
+                                                                            "If nothing is found, the default dynamic library lookup is performed, see man ld.so(8) for details.")
         ("plugin,P", po::value<vector<string>>(), "List of plugin names to load in order,"
                                                   "e.g. if the file is called 'libFairMQPlugin_example.so', just list 'example' or 'd:example' here."
                                                   "To load a prelinked plugin, list 'p:example' here.");
@@ -170,29 +171,42 @@ auto fair::mq::PluginManager::LoadPluginDynamic(const string& pluginName) -> voi
     if (fPluginFactories.find(pluginName) == fPluginFactories.end())
     {
         auto success = false;
-        for(const auto& searchPath : SearchPaths())
-        {
-            try
-            {
-                LoadSymbols(
-                    pluginName,
-                    searchPath / ToString(LibPrefix(), pluginName),
-                    dll::load_mode::append_decorations
-                );
-
+        for (const auto& searchPath : SearchPaths()) {
+            try {
+                LoadSymbols(pluginName,
+                            searchPath / ToString(LibPrefix(), pluginName),
+                            dll::load_mode::append_decorations | dll::load_mode::rtld_global);
                 fPluginOrder.push_back(pluginName);
                 success = true;
                 break;
-            }
-            catch (boost::system::system_error& e)
-            {
-                if(string{e.what()}.find("No such file or directory") == string::npos)
-                {
-                    throw PluginLoadError(ToString("An error occurred while loading dynamic plugin ", pluginName, ": ", e.what()));
+            } catch (boost::system::system_error& e) {
+                if (string{e.what()}.find("No such file or directory") == string::npos) {
+                    throw PluginLoadError(
+                        ToString("An error occurred while loading dynamic plugin ",
+                                 pluginName, ": ", e.what()));
                 }
             }
         }
-        if(!success) { throw PluginLoadError(ToString("The plugin ", pluginName, " could not be found in the plugin search paths.")); }
+
+        if (!success) {
+            try {
+                // LoadSymbols(pluginName,
+                            // ToString(LibPrefix(), pluginName),
+                            // dll::load_mode::search_system_folders | dll::load_mode::append_decorations);
+                // Not sure, why the above does not work. Workaround for now:
+                LoadSymbols(pluginName,
+                            ToString("lib",
+                                     LibPrefix(),
+                                     pluginName,
+                                     boost::dll::detail::shared_library_impl::suffix().native()),
+                            dll::load_mode::search_system_folders | dll::load_mode::rtld_global);
+                fPluginOrder.push_back(pluginName);
+            } catch (boost::system::system_error& e) {
+                throw PluginLoadError(
+                    ToString("An error occurred while loading dynamic plugin ",
+                             pluginName, ": ", e.what()));
+            }
+        }
     }
 }
 

fairmq/PluginServices.cxx

@@ -16,11 +16,14 @@ const std::unordered_map<std::string, PluginServices::DeviceState> PluginService
     {"ERROR",               DeviceState::Error},
     {"IDLE",                DeviceState::Idle},
     {"INITIALIZING DEVICE", DeviceState::InitializingDevice},
+    {"INITIALIZED",         DeviceState::Initialized},
+    {"BINDING",             DeviceState::Binding},
+    {"BOUND",               DeviceState::Bound},
+    {"CONNECTING",          DeviceState::Connecting},
     {"DEVICE READY",        DeviceState::DeviceReady},
     {"INITIALIZING TASK",   DeviceState::InitializingTask},
     {"READY",               DeviceState::Ready},
     {"RUNNING",             DeviceState::Running},
-    {"PAUSED",              DeviceState::Paused},
     {"RESETTING TASK",      DeviceState::ResettingTask},
     {"RESETTING DEVICE",    DeviceState::ResettingDevice},
     {"EXITING",             DeviceState::Exiting}
@@ -30,83 +33,96 @@ const std::unordered_map<PluginServices::DeviceState, std::string, tools::HashEn
     {DeviceState::Error,              "ERROR"},
     {DeviceState::Idle,               "IDLE"},
     {DeviceState::InitializingDevice, "INITIALIZING DEVICE"},
+    {DeviceState::Initialized,        "INITIALIZED"},
+    {DeviceState::Binding,            "BINDING"},
+    {DeviceState::Bound,              "BOUND"},
+    {DeviceState::Connecting,         "CONNECTING"},
     {DeviceState::DeviceReady,        "DEVICE READY"},
     {DeviceState::InitializingTask,   "INITIALIZING TASK"},
     {DeviceState::Ready,              "READY"},
     {DeviceState::Running,            "RUNNING"},
-    {DeviceState::Paused,             "PAUSED"},
     {DeviceState::ResettingTask,      "RESETTING TASK"},
     {DeviceState::ResettingDevice,    "RESETTING DEVICE"},
     {DeviceState::Exiting,            "EXITING"}
 };
 const std::unordered_map<std::string, PluginServices::DeviceStateTransition> PluginServices::fkDeviceStateTransitionStrMap = {
-    {"INIT DEVICE",  DeviceStateTransition::InitDevice},
-    {"INIT TASK",    DeviceStateTransition::InitTask},
-    {"RUN",          DeviceStateTransition::Run},
-    {"PAUSE",        DeviceStateTransition::Pause},
-    {"RESUME",       DeviceStateTransition::Resume},
-    {"STOP",         DeviceStateTransition::Stop},
-    {"RESET TASK",   DeviceStateTransition::ResetTask},
-    {"RESET DEVICE", DeviceStateTransition::ResetDevice},
-    {"END",          DeviceStateTransition::End},
-    {"ERROR FOUND",  DeviceStateTransition::ErrorFound},
+    {"AUTO",          DeviceStateTransition::Auto},
+    {"INIT DEVICE",   DeviceStateTransition::InitDevice},
+    {"COMPLETE INIT", DeviceStateTransition::CompleteInit},
+    {"BIND",          DeviceStateTransition::Bind},
+    {"CONNECT",       DeviceStateTransition::Connect},
+    {"INIT TASK",     DeviceStateTransition::InitTask},
+    {"RUN",           DeviceStateTransition::Run},
+    {"STOP",          DeviceStateTransition::Stop},
+    {"RESET TASK",    DeviceStateTransition::ResetTask},
+    {"RESET DEVICE",  DeviceStateTransition::ResetDevice},
+    {"END",           DeviceStateTransition::End},
+    {"ERROR FOUND",   DeviceStateTransition::ErrorFound},
 };
 const std::unordered_map<PluginServices::DeviceStateTransition, std::string, tools::HashEnum<PluginServices::DeviceStateTransition>> PluginServices::fkStrDeviceStateTransitionMap = {
-    {DeviceStateTransition::InitDevice,  "INIT DEVICE"},
-    {DeviceStateTransition::InitTask,    "INIT TASK"},
-    {DeviceStateTransition::Run,         "RUN"},
-    {DeviceStateTransition::Pause,       "PAUSE"},
-    {DeviceStateTransition::Resume,      "RESUME"},
-    {DeviceStateTransition::Stop,        "STOP"},
-    {DeviceStateTransition::ResetTask,   "RESET TASK"},
-    {DeviceStateTransition::ResetDevice, "RESET DEVICE"},
-    {DeviceStateTransition::End,         "END"},
-    {DeviceStateTransition::ErrorFound,  "ERROR FOUND"},
+    {DeviceStateTransition::Auto,         "Auto"},
+    {DeviceStateTransition::InitDevice,   "INIT DEVICE"},
+    {DeviceStateTransition::CompleteInit, "COMPLETE INIT"},
+    {DeviceStateTransition::Bind,         "BIND"},
+    {DeviceStateTransition::Connect,      "CONNECT"},
+    {DeviceStateTransition::InitTask,     "INIT TASK"},
+    {DeviceStateTransition::Run,          "RUN"},
+    {DeviceStateTransition::Stop,         "STOP"},
+    {DeviceStateTransition::ResetTask,    "RESET TASK"},
+    {DeviceStateTransition::ResetDevice,  "RESET DEVICE"},
+    {DeviceStateTransition::End,          "END"},
+    {DeviceStateTransition::ErrorFound,   "ERROR FOUND"},
 };
-const std::unordered_map<FairMQDevice::State, PluginServices::DeviceState, fair::mq::tools::HashEnum<FairMQDevice::State>> PluginServices::fkDeviceStateMap = {
-    {FairMQDevice::OK,                  DeviceState::Ok},
-    {FairMQDevice::Error,               DeviceState::Error},
-    {FairMQDevice::IDLE,                DeviceState::Idle},
-    {FairMQDevice::INITIALIZING_DEVICE, DeviceState::InitializingDevice},
-    {FairMQDevice::DEVICE_READY,        DeviceState::DeviceReady},
-    {FairMQDevice::INITIALIZING_TASK,   DeviceState::InitializingTask},
-    {FairMQDevice::READY,               DeviceState::Ready},
-    {FairMQDevice::RUNNING,             DeviceState::Running},
-    {FairMQDevice::PAUSED,              DeviceState::Paused},
-    {FairMQDevice::RESETTING_TASK,      DeviceState::ResettingTask},
-    {FairMQDevice::RESETTING_DEVICE,    DeviceState::ResettingDevice},
-    {FairMQDevice::EXITING,             DeviceState::Exiting}
+const std::unordered_map<fair::mq::State, PluginServices::DeviceState, fair::mq::tools::HashEnum<fair::mq::State>> PluginServices::fkDeviceStateMap = {
+    {fair::mq::State::Ok,                 DeviceState::Ok},
+    {fair::mq::State::Error,              DeviceState::Error},
+    {fair::mq::State::Idle,               DeviceState::Idle},
+    {fair::mq::State::InitializingDevice, DeviceState::InitializingDevice},
+    {fair::mq::State::Initialized,        DeviceState::Initialized},
+    {fair::mq::State::Binding,            DeviceState::Binding},
+    {fair::mq::State::Bound,              DeviceState::Bound},
+    {fair::mq::State::Connecting,         DeviceState::Connecting},
+    {fair::mq::State::DeviceReady,        DeviceState::DeviceReady},
+    {fair::mq::State::InitializingTask,   DeviceState::InitializingTask},
+    {fair::mq::State::Ready,              DeviceState::Ready},
+    {fair::mq::State::Running,            DeviceState::Running},
+    {fair::mq::State::ResettingTask,      DeviceState::ResettingTask},
+    {fair::mq::State::ResettingDevice,    DeviceState::ResettingDevice},
+    {fair::mq::State::Exiting,            DeviceState::Exiting}
 };
-const std::unordered_map<PluginServices::DeviceStateTransition, FairMQDevice::Event, tools::HashEnum<PluginServices::DeviceStateTransition>> PluginServices::fkDeviceStateTransitionMap = {
-    {DeviceStateTransition::InitDevice,  FairMQDevice::INIT_DEVICE},
-    {DeviceStateTransition::InitTask,    FairMQDevice::INIT_TASK},
-    {DeviceStateTransition::Run,         FairMQDevice::RUN},
-    {DeviceStateTransition::Pause,       FairMQDevice::PAUSE},
-    {DeviceStateTransition::Resume,      FairMQDevice::RUN},
-    {DeviceStateTransition::Stop,        FairMQDevice::STOP},
-    {DeviceStateTransition::ResetTask,   FairMQDevice::RESET_TASK},
-    {DeviceStateTransition::ResetDevice, FairMQDevice::RESET_DEVICE},
-    {DeviceStateTransition::End,         FairMQDevice::END},
-    {DeviceStateTransition::ErrorFound,  FairMQDevice::ERROR_FOUND}
+const std::unordered_map<PluginServices::DeviceStateTransition, fair::mq::Transition, tools::HashEnum<PluginServices::DeviceStateTransition>> PluginServices::fkDeviceStateTransitionMap = {
+    {DeviceStateTransition::Auto,         fair::mq::Transition::Auto},
+    {DeviceStateTransition::InitDevice,   fair::mq::Transition::InitDevice},
+    {DeviceStateTransition::CompleteInit, fair::mq::Transition::CompleteInit},
+    {DeviceStateTransition::Bind,         fair::mq::Transition::Bind},
+    {DeviceStateTransition::Connect,      fair::mq::Transition::Connect},
+    {DeviceStateTransition::InitTask,     fair::mq::Transition::InitTask},
+    {DeviceStateTransition::Run,          fair::mq::Transition::Run},
+    {DeviceStateTransition::Stop,         fair::mq::Transition::Stop},
+    {DeviceStateTransition::ResetTask,    fair::mq::Transition::ResetTask},
+    {DeviceStateTransition::ResetDevice,  fair::mq::Transition::ResetDevice},
+    {DeviceStateTransition::End,          fair::mq::Transition::End},
+    {DeviceStateTransition::ErrorFound,   fair::mq::Transition::ErrorFound}
 };
 
-auto PluginServices::ChangeDeviceState(const std::string& controller, const DeviceStateTransition next) -> void
+auto PluginServices::ChangeDeviceState(const std::string& controller, const DeviceStateTransition next) -> bool
 {
     lock_guard<mutex> lock{fDeviceControllerMutex};
 
     if (!fDeviceController) fDeviceController = controller;
 
-    if (fDeviceController == controller)
-    {
-        fDevice.ChangeState(fkDeviceStateTransitionMap.at(next));
-    }
-    else
-    {
+    bool result = false;
+
+    if (fDeviceController == controller) {
+        result = fDevice.ChangeState(fkDeviceStateTransitionMap.at(next));
+    } else {
         throw DeviceControlError{tools::ToString(
             "Plugin '", controller, "' is not allowed to change device states. ",
             "Currently, plugin '", *fDeviceController, "' has taken control."
         )};
     }
+
+    return result;
 }
 
 auto PluginServices::TakeDeviceControl(const std::string& controller) -> void

fairmq/PluginServices.h

@@ -63,11 +63,14 @@ class PluginServices
         Error,
         Idle,
         InitializingDevice,
+        Initialized,
+        Binding,
+        Bound,
+        Connecting,
         DeviceReady,
         InitializingTask,
         Ready,
         Running,
-        Paused,
         ResettingTask,
         ResettingDevice,
         Exiting
@@ -75,11 +78,13 @@ class PluginServices
 
     enum class DeviceStateTransition : int // transition event between DeviceStates
     {
+        Auto,
         InitDevice,
+        CompleteInit,
+        Bind,
+        Connect,
         InitTask,
         Run,
-        Pause,
-        Resume,
         Stop,
         ResetTask,
         ResetDevice,
@@ -115,7 +120,7 @@ class PluginServices
     friend auto operator<<(std::ostream& os, const DeviceStateTransition& transition) -> std::ostream& { return os << ToStr(transition); }
 
     /// @return current device state
-    auto GetCurrentDeviceState() const -> DeviceState { return fkDeviceStateMap.at(static_cast<FairMQDevice::State>(fDevice.GetCurrentState())); }
+    auto GetCurrentDeviceState() const -> DeviceState { return fkDeviceStateMap.at(static_cast<fair::mq::State>(fDevice.GetCurrentState())); }
 
     /// @brief Become device controller
     /// @param controller id
@@ -151,7 +156,7 @@ class PluginServices
     /// The state transition may not happen immediately, but when the current state evaluates the
     /// pending transition event and terminates. In other words, the device states are scheduled cooperatively.
     /// If the device control role has not been taken yet, calling this function will take over control implicitely.
-    auto ChangeDeviceState(const std::string& controller, const DeviceStateTransition next) -> void;
+    auto ChangeDeviceState(const std::string& controller, const DeviceStateTransition next) -> bool;
 
     /// @brief Subscribe with a callback to device state changes
     /// @param subscriber id
@@ -161,7 +166,7 @@ class PluginServices
     /// the state is running in.
     auto SubscribeToDeviceStateChange(const std::string& subscriber, std::function<void(DeviceState /*newState*/)> callback) -> void
     {
-        fDevice.SubscribeToStateChange(subscriber, [&,callback](FairMQDevice::State newState){
+        fDevice.SubscribeToStateChange(subscriber, [&,callback](fair::mq::State newState){
             callback(fkDeviceStateMap.at(newState));
         });
     }
@@ -187,12 +192,13 @@ class PluginServices
     {
         auto currentState = GetCurrentDeviceState();
         if (   (currentState == DeviceState::InitializingDevice)
-            || ((currentState == DeviceState::Idle) && (key == "channel-config")))
-        {
+            || (currentState == DeviceState::Initialized)
+            || (currentState == DeviceState::Binding)
+            || (currentState == DeviceState::Bound)
+            || (currentState == DeviceState::Connecting)
+            || (currentState == DeviceState::Idle && key == "channel-config")) {
             fConfig.SetValue(key, val);
-        }
-        else
-        {
+        } else {
             throw InvalidStateError{
                 tools::ToString("PluginServices::SetProperty is not supported in device state ", currentState, ". ",
                                 "Supported state is ", DeviceState::InitializingDevice, ".")};
@@ -271,8 +277,8 @@ class PluginServices
     static const std::unordered_map<DeviceState, std::string, tools::HashEnum<DeviceState>> fkStrDeviceStateMap;
     static const std::unordered_map<std::string, DeviceStateTransition> fkDeviceStateTransitionStrMap;
     static const std::unordered_map<DeviceStateTransition, std::string, tools::HashEnum<DeviceStateTransition>> fkStrDeviceStateTransitionMap;
-    static const std::unordered_map<FairMQDevice::State, DeviceState, tools::HashEnum<FairMQDevice::State>> fkDeviceStateMap;
-    static const std::unordered_map<DeviceStateTransition, FairMQDevice::Event, tools::HashEnum<DeviceStateTransition>> fkDeviceStateTransitionMap;
+    static const std::unordered_map<fair::mq::State, DeviceState, tools::HashEnum<fair::mq::State>> fkDeviceStateMap;
+    static const std::unordered_map<DeviceStateTransition, fair::mq::Transition, tools::HashEnum<DeviceStateTransition>> fkDeviceStateTransitionMap;
 
   private:
     FairMQProgOptions& fConfig;

fairmq/StateMachine.cxx

@@ -1,5 +1,5 @@
 /********************************************************************************
- *    Copyright (C) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -7,189 +7,476 @@
  ********************************************************************************/
 
 #include "StateMachine.h"
+#include <fairmq/Tools.h>
+
+// Increase maximum number of boost::msm states (default is 10)
+// This #define has to be before any msm header includes
+#define FUSION_MAX_VECTOR_SIZE 20
+
+#include <boost/mpl/for_each.hpp>
+#include <boost/msm/back/state_machine.hpp>
+#include <boost/msm/back/tools.hpp>
+#include <boost/msm/back/metafunctions.hpp>
+#include <boost/msm/front/state_machine_def.hpp>
+#include <boost/msm/front/functor_row.hpp>
+#include <boost/core/demangle.hpp>
+#include <boost/signals2.hpp> // signal/slot for onStateChange callbacks
+
+#include <atomic>
+#include <condition_variable>
+#include <chrono>
+#include <array>
+#include <unordered_map>
+#include <mutex>
 
-using namespace fair::mq;
 using namespace std;
+using namespace boost::msm;
+using namespace boost::msm::front;
+using namespace boost::msm::back;
+namespace bmpl = boost::mpl;
 
-const std::unordered_map<std::string, StateMachine::State> StateMachine::fkStateStrMap = {
-    {"OK",                  State::Ok},
-    {"ERROR",               State::Error},
-    {"IDLE",                State::Idle},
-    {"INITIALIZING DEVICE", State::InitializingDevice},
-    {"DEVICE READY",        State::DeviceReady},
-    {"INITIALIZING TASK",   State::InitializingTask},
-    {"READY",               State::Ready},
-    {"RUNNING",             State::Running},
-    {"RESETTING TASK",      State::ResettingTask},
-    {"RESETTING DEVICE",    State::ResettingDevice},
-    {"EXITING",             State::Exiting}
-};
-const std::unordered_map<StateMachine::State, std::string, tools::HashEnum<StateMachine::State>> StateMachine::fkStrStateMap = {
-    {State::Ok,                 "OK"},
-    {State::Error,              "ERROR"},
-    {State::Idle,               "IDLE"},
-    {State::InitializingDevice, "INITIALIZING DEVICE"},
-    {State::DeviceReady,        "DEVICE READY"},
-    {State::InitializingTask,   "INITIALIZING TASK"},
-    {State::Ready,              "READY"},
-    {State::Running,            "RUNNING"},
-    {State::ResettingTask,      "RESETTING TASK"},
-    {State::ResettingDevice,    "RESETTING DEVICE"},
-    {State::Exiting,            "EXITING"}
-};
-const std::unordered_map<std::string, StateMachine::StateTransition> StateMachine::fkStateTransitionStrMap = {
-    {"INIT DEVICE",  StateTransition::InitDevice},
-    {"INIT TASK",    StateTransition::InitTask},
-    {"RUN",          StateTransition::Run},
-    {"STOP",         StateTransition::Stop},
-    {"RESET TASK",   StateTransition::ResetTask},
-    {"RESET DEVICE", StateTransition::ResetDevice},
-    {"END",          StateTransition::End},
-    {"ERROR FOUND",  StateTransition::ErrorFound},
-    {"AUTOMATIC",    StateTransition::Automatic},
-};
-const std::unordered_map<StateMachine::StateTransition, std::string, tools::HashEnum<StateMachine::StateTransition>> StateMachine::fkStrStateTransitionMap = {
-    {StateTransition::InitDevice,  "INIT DEVICE"},
-    {StateTransition::InitTask,    "INIT TASK"},
-    {StateTransition::Run,         "RUN"},
-    {StateTransition::Stop,        "STOP"},
-    {StateTransition::ResetTask,   "RESET TASK"},
-    {StateTransition::ResetDevice, "RESET DEVICE"},
-    {StateTransition::End,         "END"},
-    {StateTransition::ErrorFound,  "ERROR FOUND"},
-    {StateTransition::Automatic,   "AUTOMATIC"},
-};
-
-auto StateMachine::Run() -> void
+namespace std
 {
-    LOG(state) << "Starting FairMQ state machine";
 
-    LOG(debug) << "Entering initial " << fErrorState << " state (orthogonal error state machine)";
-    LOG(state) << "Entering initial " << fState << " state";
+template<>
+struct hash<fair::mq::Transition> : fair::mq::tools::HashEnum<fair::mq::Transition> {};
 
-    std::unique_lock<std::mutex> lock{fMutex};
-    while (true)
+template<>
+struct hash<fair::mq::State> : fair::mq::tools::HashEnum<fair::mq::State> {};
+
+} /* namespace std */
+
+namespace fair
+{
+namespace mq
+{
+namespace fsm
+{
+
+// list of FSM states
+struct OK_S                  : public state<> { static string Name() { return "OK"; }                  static State Type() { return State::Ok; } };
+
+struct IDLE_S                : public state<> { static string Name() { return "IDLE"; }                static State Type() { return State::Idle; } };
+struct INITIALIZING_DEVICE_S : public state<> { static string Name() { return "INITIALIZING_DEVICE"; } static State Type() { return State::InitializingDevice; } };
+struct INITIALIZED_S         : public state<> { static string Name() { return "INITIALIZED"; }         static State Type() { return State::Initialized; } };
+struct BINDING_S             : public state<> { static string Name() { return "BINDING"; }             static State Type() { return State::Binding; } };
+struct BOUND_S               : public state<> { static string Name() { return "BOUND"; }               static State Type() { return State::Bound; } };
+struct CONNECTING_S          : public state<> { static string Name() { return "CONNECTING"; }          static State Type() { return State::Connecting; } };
+struct DEVICE_READY_S        : public state<> { static string Name() { return "DEVICE_READY"; }        static State Type() { return State::DeviceReady; } };
+struct INITIALIZING_TASK_S   : public state<> { static string Name() { return "INITIALIZING_TASK"; }   static State Type() { return State::InitializingTask; } };
+struct READY_S               : public state<> { static string Name() { return "READY"; }               static State Type() { return State::Ready; } };
+struct RUNNING_S             : public state<> { static string Name() { return "RUNNING"; }             static State Type() { return State::Running; } };
+struct RESETTING_TASK_S      : public state<> { static string Name() { return "RESETTING_TASK"; }      static State Type() { return State::ResettingTask; } };
+struct RESETTING_DEVICE_S    : public state<> { static string Name() { return "RESETTING_DEVICE"; }    static State Type() { return State::ResettingDevice; } };
+struct EXITING_S             : public state<> { static string Name() { return "EXITING"; }             static State Type() { return State::Exiting; } };
+
+struct ERROR_S               : public terminate_state<> { static string Name() { return "ERROR"; }     static State Type() { return State::Error; } };
+
+// list of FSM transitions (events)
+struct AUTO_E          { static string Name() { return "AUTO"; }          static Transition Type() { return Transition::Auto; } };
+struct INIT_DEVICE_E   { static string Name() { return "INIT_DEVICE"; }   static Transition Type() { return Transition::InitDevice; } };
+struct COMPLETE_INIT_E { static string Name() { return "COMPLETE_INIT"; } static Transition Type() { return Transition::CompleteInit; } };
+struct BIND_E          { static string Name() { return "BIND"; }          static Transition Type() { return Transition::Bind; } };
+struct CONNECT_E       { static string Name() { return "CONNECT"; }       static Transition Type() { return Transition::Connect; } };
+struct INIT_TASK_E     { static string Name() { return "INIT_TASK"; }     static Transition Type() { return Transition::InitTask; } };
+struct RUN_E           { static string Name() { return "RUN"; }           static Transition Type() { return Transition::Run; } };
+struct STOP_E          { static string Name() { return "STOP"; }          static Transition Type() { return Transition::Stop; } };
+struct RESET_TASK_E    { static string Name() { return "RESET_TASK"; }    static Transition Type() { return Transition::ResetTask; } };
+struct RESET_DEVICE_E  { static string Name() { return "RESET_DEVICE"; }  static Transition Type() { return Transition::ResetDevice; } };
+struct END_E           { static string Name() { return "END"; }           static Transition Type() { return Transition::End; } };
+struct ERROR_FOUND_E   { static string Name() { return "ERROR_FOUND"; }   static Transition Type() { return Transition::ErrorFound; } };
+
+static array<string, 15> stateNames =
+{
     {
-        while (fNextStates.empty())
+        "OK",
+        "Error",
+        "IDLE",
+        "INITIALIZING_DEVICE",
+        "INITIALIZED",
+        "BINDING",
+        "BOUND",
+        "CONNECTING",
+        "DEVICE_READY",
+        "INITIALIZING_TASK",
+        "READY",
+        "RUNNING",
+        "RESETTING_TASK",
+        "RESETTING_DEVICE",
+        "EXITING"
+    }
+};
+
+static array<string, 12> transitionNames =
+{
+    {
+        "AUTO",
+        "INIT_DEVICE",
+        "COMPLETE_INIT",
+        "BIND",
+        "CONNECT",
+        "INIT_TASK",
+        "RUN",
+        "STOP",
+        "RESET_TASK",
+        "RESET_DEVICE",
+        "END",
+        "ERROR_FOUND"
+    }
+};
+
+static map<string, State> stateNumbers =
+{
+    { "OK",                  State::Ok },
+    { "Error",               State::Error },
+    { "IDLE",                State::Idle },
+    { "INITIALIZING_DEVICE", State::InitializingDevice },
+    { "INITIALIZED",         State::Initialized },
+    { "BINDING",             State::Binding },
+    { "BOUND",               State::Bound },
+    { "CONNECTING",          State::Connecting },
+    { "DEVICE_READY",        State::DeviceReady },
+    { "INITIALIZING_TASK",   State::InitializingTask },
+    { "READY",               State::Ready },
+    { "RUNNING",             State::Running },
+    { "RESETTING_TASK",      State::ResettingTask },
+    { "RESETTING_DEVICE",    State::ResettingDevice },
+    { "EXITING",             State::Exiting }
+};
+
+static map<string, Transition> transitionNumbers =
+{
+    { "AUTO",          Transition::Auto },
+    { "INIT_DEVICE",   Transition::InitDevice },
+    { "COMPLETE_INIT", Transition::CompleteInit },
+    { "BIND",          Transition::Bind },
+    { "CONNECT",       Transition::Connect },
+    { "INIT_TASK",     Transition::InitTask },
+    { "RUN",           Transition::Run },
+    { "STOP",          Transition::Stop },
+    { "RESET_TASK",    Transition::ResetTask },
+    { "RESET_DEVICE",  Transition::ResetDevice },
+    { "END",           Transition::End },
+    { "ERROR_FOUND",   Transition::ErrorFound }
+};
+
+// defining the boost MSM state machine
+struct Machine_ : public state_machine_def<Machine_>
+{
+  public:
+    Machine_()
+        : fLastTransitionResult(true)
+        , fNewStatePending(false)
+        , fWorkOngoing(false)
+    {}
+
+    virtual ~Machine_() {}
+
+    // initial states
+    using initial_state = bmpl::vector<IDLE_S, OK_S>;
+
+    template<typename Transition, typename FSM>
+    void on_entry(Transition const&, FSM& /* fsm */)
+    {
+        LOG(state) << "Starting FairMQ state machine --> IDLE";
+        fState = State::Idle;
+    }
+
+    template<typename Transition, typename FSM>
+    void on_exit(Transition const&, FSM& /*fsm*/)
+    {
+        LOG(state) << "Exiting FairMQ state machine";
+    }
+
+    struct DefaultFct
+    {
+        template<typename EVT, typename FSM, typename SourceState, typename TargetState>
+        void operator()(EVT const& e, FSM& fsm, SourceState& /* ss */, TargetState& ts)
         {
-            fNewState.wait(lock);
+            fsm.fNewState = ts.Type();
+            fsm.fLastTransitionResult = true;
+            fsm.CallNewTransitionCallbacks(e.Type());
+            fsm.fNewStatePending = true;
+            fsm.fNewStatePendingCV.notify_all();
         }
+    };
 
-        State lastState;
+    struct transition_table : bmpl::vector<
+        //  Start                  Transition       Next                   Action      Guard
+        Row<IDLE_S,                END_E,           EXITING_S,             DefaultFct, none>,
+        Row<IDLE_S,                INIT_DEVICE_E,   INITIALIZING_DEVICE_S, DefaultFct, none>,
 
-        if (fNextStates.front() == State::Error)
+        Row<INITIALIZING_DEVICE_S, COMPLETE_INIT_E, INITIALIZED_S,         DefaultFct, none>,
+        Row<INITIALIZED_S,         BIND_E,          BINDING_S,             DefaultFct, none>,
+        Row<INITIALIZED_S,         RESET_DEVICE_E,  RESETTING_DEVICE_S,    DefaultFct, none>,
+
+        Row<BINDING_S,             AUTO_E,          BOUND_S,               DefaultFct, none>,
+        Row<BOUND_S,               CONNECT_E,       CONNECTING_S,          DefaultFct, none>,
+        Row<BOUND_S,               RESET_DEVICE_E,  RESETTING_DEVICE_S,    DefaultFct, none>,
+
+        Row<CONNECTING_S,          AUTO_E,          DEVICE_READY_S,        DefaultFct, none>,
+        Row<DEVICE_READY_S,        INIT_TASK_E,     INITIALIZING_TASK_S,   DefaultFct, none>,
+        Row<DEVICE_READY_S,        RESET_DEVICE_E,  RESETTING_DEVICE_S,    DefaultFct, none>,
+
+        Row<INITIALIZING_TASK_S,   AUTO_E,          READY_S,               DefaultFct, none>,
+
+        Row<READY_S,               RUN_E,           RUNNING_S,             DefaultFct, none>,
+        Row<READY_S,               RESET_TASK_E,    RESETTING_TASK_S,      DefaultFct, none>,
+
+        Row<RUNNING_S,             STOP_E,          READY_S,               DefaultFct, none>,
+
+        Row<RESETTING_TASK_S,      AUTO_E,          DEVICE_READY_S,        DefaultFct, none>,
+        Row<RESETTING_DEVICE_S,    AUTO_E,          IDLE_S,                DefaultFct, none>,
+
+        Row<OK_S,                  ERROR_FOUND_E,   ERROR_S,               DefaultFct,   none>> {};
+
+    void CallStateChangeCallbacks(const State state) const
+    {
+        if (!fStateChangeSignal.empty()) {
+            fStateChangeSignal(state);
+        }
+    }
+
+    void CallStateHandler(const State state) const
+    {
+        if (!fStateHandleSignal.empty()) {
+            fStateHandleSignal(state);
+        }
+    }
+
+    void CallNewTransitionCallbacks(const Transition transition) const
+    {
+        if (!fNewTransitionSignal.empty()) {
+            fNewTransitionSignal(transition);
+        }
+    }
+
+
+    atomic<State> fState;
+    atomic<State> fNewState;
+    atomic<bool> fLastTransitionResult;
+
+    mutex fStateMtx;
+    atomic<bool> fNewStatePending;
+    atomic<bool> fWorkOngoing;
+    condition_variable fNewStatePendingCV;
+    condition_variable fWorkDoneCV;
+
+    boost::signals2::signal<void(const State)> fStateChangeSignal;
+    boost::signals2::signal<void(const State)> fStateHandleSignal;
+    boost::signals2::signal<void(const Transition)> fNewTransitionSignal;
+    unordered_map<string, boost::signals2::connection> fStateChangeSignalsMap;
+    unordered_map<string, boost::signals2::connection> fNewTransitionSignalsMap;
+
+    void ProcessWork()
+    {
+        bool stop = false;
+
+        while (!stop) {
+            {
+                unique_lock<mutex> lock(fStateMtx);
+
+                while (!fNewStatePending) {
+                    fNewStatePendingCV.wait_for(lock, chrono::milliseconds(100));
+                }
+
+                LOG(state) << fState << " ---> " << fNewState;
+                fState = static_cast<State>(fNewState);
+                fNewStatePending = false;
+                fWorkOngoing = true;
+
+                if (fState == State::Exiting || fState == State::Error) {
+                    stop = true;
+                }
+            }
+
+            CallStateChangeCallbacks(fState);
+            CallStateHandler(fState);
+
+            {
+                lock_guard<mutex> lock(fStateMtx);
+                fWorkOngoing = false;
+                fWorkDoneCV.notify_one();
+            }
+        }
+    }
+
+    // replaces the default no-transition response.
+    template<typename FSM, typename Transition>
+    void no_transition(Transition const& t, FSM& fsm, int state)
+    {
+        using RecursiveStt = typename recursive_get_transition_table<FSM>::type;
+        using AllStates = typename generate_state_set<RecursiveStt>::type;
+
+        string stateName;
+
+        bmpl::for_each<AllStates, wrap<bmpl::placeholders::_1>>(get_state_name<RecursiveStt>(stateName, state));
+
+        stateName = boost::core::demangle(stateName.c_str());
+        size_t pos = stateName.rfind(":");
+        stateName = stateName.substr(pos + 1);
+        size_t pos2 = stateName.rfind("_");
+        stateName = stateName.substr(0, pos2);
+
+        if (stateName != "OK") {
+            LOG(state) << "No transition from state " << stateName << " on transition " << t.Name();
+        }
+        fsm.fLastTransitionResult = false;
+    }
+}; // Machine_
+
+using FairMQFSM = state_machine<Machine_>;
+
+} // namespace fsm
+} // namespace mq
+} // namespace fair
+
+using namespace fair::mq::fsm;
+using namespace fair::mq;
+
+StateMachine::StateMachine() : fFsm(new FairMQFSM) {}
+void StateMachine::Start() { static_pointer_cast<FairMQFSM>(fFsm)->start(); }
+StateMachine::~StateMachine() { static_pointer_cast<FairMQFSM>(fFsm)->stop(); }
+
+bool StateMachine::ChangeState(const Transition transition)
+try {
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    lock_guard<mutex> lock(fsm->fStateMtx);
+    if (!static_cast<bool>(fsm->fNewStatePending) || transition == Transition::ErrorFound) {
+        switch (transition) {
+            case Transition::Auto:
+                fsm->process_event(AUTO_E());
+                return fsm->fLastTransitionResult;
+            case Transition::InitDevice:
+                fsm->process_event(INIT_DEVICE_E());
+                return fsm->fLastTransitionResult;
+            case Transition::CompleteInit:
+                fsm->process_event(COMPLETE_INIT_E());
+                return fsm->fLastTransitionResult;
+            case Transition::Bind:
+                fsm->process_event(BIND_E());
+                return fsm->fLastTransitionResult;
+            case Transition::Connect:
+                fsm->process_event(CONNECT_E());
+                return fsm->fLastTransitionResult;
+            case Transition::InitTask:
+                fsm->process_event(INIT_TASK_E());
+                return fsm->fLastTransitionResult;
+            case Transition::Run:
+                fsm->process_event(RUN_E());
+                return fsm->fLastTransitionResult;
+            case Transition::Stop:
+                fsm->process_event(STOP_E());
+                return fsm->fLastTransitionResult;
+            case Transition::ResetDevice:
+                fsm->process_event(RESET_DEVICE_E());
+                return fsm->fLastTransitionResult;
+            case Transition::ResetTask:
+                fsm->process_event(RESET_TASK_E());
+                return fsm->fLastTransitionResult;
+            case Transition::End:
+                fsm->process_event(END_E());
+                return fsm->fLastTransitionResult;
+            case Transition::ErrorFound:
+                fsm->process_event(ERROR_FOUND_E());
+                return fsm->fLastTransitionResult;
+            default:
+                LOG(error) << "Requested unsupported state transition: " << transition << endl;
+                return false;
+             }
+    } else {
+        LOG(state) << "Transition " << transitionNames.at(static_cast<int>(transition)) << " incoming, but another state transition is already ongoing.";
+        return false;
+    }
+} catch (exception& e) {
+    LOG(error) << "Exception in StateMachine::ChangeState(): " << e.what();
+    return false;
+}
+
+void StateMachine::SubscribeToStateChange(const string& key, function<void(const State)> callback)
+{
+    static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignalsMap.insert({key, static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignal.connect(callback)});
+}
+
+void StateMachine::UnsubscribeFromStateChange(const string& key)
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    if (fsm->fStateChangeSignalsMap.count(key)) {
+        fsm->fStateChangeSignalsMap.at(key).disconnect();
+        fsm->fStateChangeSignalsMap.erase(key);
+    }
+}
+
+void StateMachine::HandleStates(function<void(const State)> callback)
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    if (fsm->fStateHandleSignal.empty()) {
+        fsm->fStateHandleSignal.connect(callback);
+    } else {
+        LOG(error) << "state handler is already set";
+    }
+}
+
+void StateMachine::StopHandlingStates()
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    if (!fsm->fStateHandleSignal.empty()) {
+        fsm->fStateHandleSignal.disconnect_all_slots();
+    }
+}
+
+void StateMachine::SubscribeToNewTransition(const string& key, function<void(const Transition)> callback)
+{
+    static_pointer_cast<FairMQFSM>(fFsm)->fNewTransitionSignalsMap.insert({key, static_pointer_cast<FairMQFSM>(fFsm)->fNewTransitionSignal.connect(callback)});
+}
+
+void StateMachine::UnsubscribeFromNewTransition(const string& key)
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    if (fsm->fNewTransitionSignalsMap.count(key)) {
+        fsm->fNewTransitionSignalsMap.at(key).disconnect();
+        fsm->fNewTransitionSignalsMap.erase(key);
+    }
+}
+
+State StateMachine::GetCurrentState() const { return static_pointer_cast<FairMQFSM>(fFsm)->fState; }
+string StateMachine::GetCurrentStateName() const { return GetStateName(static_pointer_cast<FairMQFSM>(fFsm)->fState); }
+
+bool StateMachine::NewStatePending() const { return static_cast<bool>(static_pointer_cast<FairMQFSM>(fFsm)->fNewStatePending); }
+void StateMachine::WaitForPendingState() const
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    unique_lock<mutex> lock(fsm->fStateMtx);
+    fsm->fNewStatePendingCV.wait(lock, [&]{ return static_cast<bool>(fsm->fNewStatePending); });
+}
+bool StateMachine::WaitForPendingStateFor(const int durationInMs) const
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    unique_lock<mutex> lock(fsm->fStateMtx);
+    return fsm->fNewStatePendingCV.wait_for(lock, std::chrono::milliseconds(durationInMs), [&]{ return static_cast<bool>(fsm->fNewStatePending); });
+}
+
+void StateMachine::ProcessWork()
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+
+    try {
+        fsm->CallStateChangeCallbacks(State::Idle);
+        fsm->ProcessWork();
+    } catch(...) {
         {
-            // advance error FSM
-            lastState = fErrorState;
-            fErrorState = fNextStates.front();
-            fNextStates.pop_front();
-            LOG(error) << "Entering " << fErrorState << " state (orthogonal error state machine)";
+            lock_guard<mutex> lock(fsm->fStateMtx);
+            fsm->fState = State::Error;
+            fsm->CallStateChangeCallbacks(State::Error);
+            fsm->fWorkOngoing = false;
+            fsm->fWorkDoneCV.notify_one();
         }
-        else
-        {
-            // advance regular FSM
-            lastState = fState;
-            fState = fNextStates.front();
-            fNextStates.pop_front();
-            LOG(state) << "Entering " << fState << " state";
-        }
-        lock.unlock();
-
-        fCallbacks.Emit<StateChange, State>(fState, lastState);
-
-        lock.lock();
-        if (fState == State::Exiting || fErrorState == State::Error) break;
+        ChangeState(Transition::ErrorFound);
+        throw;
     }
-
-    LOG(state) << "Exiting FairMQ state machine";
 }
 
-auto StateMachine::ChangeState(StateTransition transition) -> void
-{
-    State lastState;
-
-    std::unique_lock<std::mutex> lock{fMutex};
-
-    if (transition == StateTransition::ErrorFound)
-    {
-        lastState = fErrorState;
-    }
-    else if (fNextStates.empty())
-    {
-        lastState = fState;
-    }
-    else
-    {
-        lastState = fNextStates.back();
-    }
-
-    const State nextState{Transition(lastState, transition)};
-    fNextStates.push_back(nextState);
-    lock.unlock();
-
-    fCallbacks.Emit<StateQueued, State>(nextState, lastState);
-    fNewState.notify_one();
-}
-    
-auto StateMachine::Transition(const State currentState, const StateTransition transition) -> State
-{
-    switch (currentState) {
-        case State::Idle:
-            if (transition == StateTransition::InitDevice ) return State::InitializingDevice;
-            if (transition == StateTransition::End        ) return State::Exiting;
-            break;
-        case State::InitializingDevice:
-            if (transition == StateTransition::Automatic  ) return State::DeviceReady;
-            break;
-        case State::DeviceReady:
-            if (transition == StateTransition::InitTask   ) return State::InitializingTask;
-            if (transition == StateTransition::ResetDevice) return State::ResettingDevice;
-            break;
-        case State::InitializingTask:
-            if (transition == StateTransition::Automatic  ) return State::Ready;
-            break;
-        case State::Ready:
-            if (transition == StateTransition::Run        ) return State::Running;
-            if (transition == StateTransition::ResetTask  ) return State::ResettingTask;
-            break;
-        case State::Running:
-            if (transition == StateTransition::Stop       ) return State::Ready;
-            break;
-        case State::ResettingTask:
-            if (transition == StateTransition::Automatic  ) return State::DeviceReady;
-            break;
-        case State::ResettingDevice:
-            if (transition == StateTransition::Automatic  ) return State::Idle;
-            break;
-        case State::Exiting:
-            break;
-        case State::Ok:
-            if (transition == StateTransition::ErrorFound ) return State::Error;
-            break;
-        case State::Error:
-            break;
-    }
-    throw IllegalTransition{tools::ToString("No transition ", transition, " from state ", currentState, ".")};
-}
-
-StateMachine::StateMachine()
-: fState{State::Idle}
-, fErrorState{State::Ok}
-{
-}
-
-auto StateMachine::Reset() -> void
-{
-    std::unique_lock<std::mutex> lock{fMutex};
-
-    fState = State::Idle;
-    fErrorState = State::Ok;
-    fNextStates.clear();
-}
-
-auto StateMachine::NextStatePending() -> bool
-{
-    std::unique_lock<std::mutex> lock{fMutex};
-
-    return fNextStates.size() > 0;
-}
+string StateMachine::GetStateName(const State state) { return stateNames.at(static_cast<int>(state)); }
+string StateMachine::GetTransitionName(const Transition transition) { return transitionNames.at(static_cast<int>(transition)); }
+State StateMachine::GetState(const string& state) { return stateNumbers.at(state); }
+Transition StateMachine::GetTransition(const string& transition) { return transitionNumbers.at(transition); }

fairmq/StateMachine.h

@@ -1,132 +1,107 @@
 /********************************************************************************
- *    Copyright (C) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
-#ifndef FAIR_MQ_STATEMACHINE_H
-#define FAIR_MQ_STATEMACHINE_H
+#ifndef FAIRMQSTATEMACHINE_H_
+#define FAIRMQSTATEMACHINE_H_
 
-#include <utility>
-#include <FairMQLogger.h>
-#include <fairmq/Tools.h>
-#include <fairmq/EventManager.h>
-#include <deque>
+#include "FairMQLogger.h"
+
+#include <string>
+#include <memory>
 #include <functional>
-#include <mutex>
 #include <condition_variable>
-#include <thread>
-#include <unordered_map>
+#include <ostream>
+#include <queue>
+#include <mutex>
+#include <stdexcept>
 
 namespace fair
 {
 namespace mq
 {
 
-/**
- * @class StateMachine StateMachine.h <fairmq/StateMachine.h>
- * @brief Implements the state machine for FairMQ devices
- *
- * See https://github.com/FairRootGroup/FairRoot/blob/dev/fairmq/docs/Device.md#13-state-machine
- */
+enum class State : int
+{
+    Ok,
+    Error,
+    Idle,
+    InitializingDevice,
+    Initialized,
+    Binding,
+    Bound,
+    Connecting,
+    DeviceReady,
+    InitializingTask,
+    Ready,
+    Running,
+    ResettingTask,
+    ResettingDevice,
+    Exiting
+};
+
+enum class Transition : int
+{
+    Auto,
+    InitDevice,
+    CompleteInit,
+    Bind,
+    Connect,
+    InitTask,
+    Run,
+    Stop,
+    ResetTask,
+    ResetDevice,
+    End,
+    ErrorFound
+};
+
 class StateMachine
 {
   public:
-    enum class State : int
-    {
-        Ok,
-        Error,
-        Idle,
-        InitializingDevice,
-        DeviceReady,
-        InitializingTask,
-        Ready,
-        Running,
-        ResettingTask,
-        ResettingDevice,
-        Exiting
-    };
-
-    enum class StateTransition : int // transition event between States
-    {
-        InitDevice,
-        InitTask,
-        Run,
-        Stop,
-        ResetTask,
-        ResetDevice,
-        End,
-        ErrorFound,
-        Automatic
-    };
-
-    /// @brief Convert string to State
-    /// @param state to convert
-    /// @return State enum entry
-    /// @throw std::out_of_range if a string cannot be resolved to a State
-    static auto ToState(const std::string& state) -> State { return fkStateStrMap.at(state); }
-
-    /// @brief Convert string to StateTransition
-    /// @param transition to convert
-    /// @return StateTransition enum entry
-    /// @throw std::out_of_range if a string cannot be resolved to a StateTransition
-    static auto ToStateTransition(const std::string& transition) -> StateTransition { return fkStateTransitionStrMap.at(transition); }
-
-    /// @brief Convert State to string
-    /// @param state to convert
-    /// @return string representation of State enum entry
-    static auto ToStr(State state) -> std::string { return fkStrStateMap.at(state); }
-
-    /// @brief Convert StateTransition to string
-    /// @param transition to convert
-    /// @return string representation of StateTransition enum entry
-    static auto ToStr(StateTransition transition) -> std::string { return fkStrStateTransitionMap.at(transition); }
-
-    friend auto operator<<(std::ostream& os, const State& state) -> std::ostream& { return os << ToStr(state); }
-    friend auto operator<<(std::ostream& os, const StateTransition& transition) -> std::ostream& { return os << ToStr(transition); }
-
     StateMachine();
+    virtual ~StateMachine();
 
-    struct IllegalTransition : std::runtime_error { using std::runtime_error::runtime_error; };
+    bool ChangeState(const Transition transition);
+    bool ChangeState(const std::string& transition) { return ChangeState(GetTransition(transition)); }
 
-    struct StateChange : Event<State> {};
-    struct StateQueued : Event<State> {};
-    auto SubscribeToStateChange(const std::string& subscriber, std::function<void(typename StateChange::KeyType newState, State lastState)> callback) -> void { fCallbacks.Subscribe<StateChange, State>(subscriber, callback); }
-    auto UnsubscribeFromStateChange(const std::string& subscriber) -> void { fCallbacks.Unsubscribe<StateChange, State>(subscriber); }
-    auto SubscribeToStateQueued(const std::string& subscriber, std::function<void(typename StateQueued::KeyType newState, State lastState)> callback) -> void { fCallbacks.Subscribe<StateQueued, State>(subscriber, callback); }
-    auto UnsubscribeFromStateQueued(const std::string& subscriber) -> void { fCallbacks.Unsubscribe<StateQueued, State>(subscriber); }
+    void SubscribeToStateChange(const std::string& key, std::function<void(const State)> callback);
+    void UnsubscribeFromStateChange(const std::string& key);
 
-    auto GetCurrentState() const -> State { std::lock_guard<std::mutex> lock{fMutex}; return fState; }
-    auto GetCurrentErrorState() const -> State { std::lock_guard<std::mutex> lock{fMutex}; return fErrorState; }
-    auto GetLastQueuedState() const -> State { std::lock_guard<std::mutex> lock{fMutex}; return fNextStates.back(); }
+    void HandleStates(std::function<void(const State)> callback);
+    void StopHandlingStates();
 
-    auto ChangeState(StateTransition transition) -> void;
+    void SubscribeToNewTransition(const std::string& key, std::function<void(const Transition)> callback);
+    void UnsubscribeFromNewTransition(const std::string& key);
 
-    auto Run() -> void;
-    auto Reset() -> void;
+    bool NewStatePending() const;
+    void WaitForPendingState() const;
+    bool WaitForPendingStateFor(const int durationInMs) const;
 
-    auto NextStatePending() -> bool;
+    State GetCurrentState() const;
+    std::string GetCurrentStateName() const;
+
+    void Start();
+
+    void ProcessWork();
+
+    static std::string GetStateName(const State);
+    static std::string GetTransitionName(const Transition);
+    static State GetState(const std::string& state);
+    static Transition GetTransition(const std::string& transition);
 
   private:
-    State fState;
-    State fErrorState;
-    std::deque<State> fNextStates;
-    EventManager fCallbacks;
+    std::shared_ptr<void> fFsm;
+};
 
-    static const std::unordered_map<std::string, State> fkStateStrMap;
-    static const std::unordered_map<State, std::string, tools::HashEnum<State>> fkStrStateMap;
-    static const std::unordered_map<std::string, StateTransition> fkStateTransitionStrMap;
-    static const std::unordered_map<StateTransition, std::string, tools::HashEnum<StateTransition>> fkStrStateTransitionMap;
+inline std::ostream& operator<<(std::ostream& os, const State& state) { return os << StateMachine::GetStateName(state); }
+inline std::ostream& operator<<(std::ostream& os, const Transition& transition) { return os << StateMachine::GetTransitionName(transition); }
 
-    mutable std::mutex fMutex;
-    std::condition_variable fNewState;
+} // namespace mq
+} // namespace fair
 
-    static auto Transition(const State currentState, const StateTransition transition) -> State;
-}; /* class StateMachine */
-
-} /* namespace mq */
-} /* namespace fair */
-
-#endif /* FAIR_MQ_STATEMACHINE_H */
+#endif /* FAIRMQSTATEMACHINE_H_ */

fairmq/apply-clang-format.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-find . -type f \( -iname "*.h" ! -iname "*.pb.h" ! -iname "*LinkDef.h" -o -iname "*.cxx" -o -iname "*.tpl" \) -execdir clang-format -i {} \;

fairmq/devices/FairMQBenchmarkSampler.cxx

@@ -54,7 +54,7 @@ void FairMQBenchmarkSampler::Run()
 
     fair::mq::tools::RateLimiter rateLimiter(fMsgRate);
 
-    while (CheckCurrentState(RUNNING))
+    while (!NewStatePending())
     {
         if (fMultipart)
         {

fairmq/devices/FairMQMerger.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**
@@ -60,7 +60,7 @@ void FairMQMerger::Run()
 
     if (fMultipart)
     {
-        while (CheckCurrentState(RUNNING))
+        while (!NewStatePending())
         {
             poller->Poll(100);
 
@@ -91,7 +91,7 @@ void FairMQMerger::Run()
     }
     else
     {
-        while (CheckCurrentState(RUNNING))
+        while (!NewStatePending())
         {
             poller->Poll(100);
 

fairmq/devices/FairMQMerger.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/devices/FairMQMultiplier.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/devices/FairMQMultiplier.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/devices/FairMQProxy.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**
@@ -41,7 +41,7 @@ void FairMQProxy::Run()
 {
     if (fMultipart)
     {
-        while (CheckCurrentState(RUNNING))
+        while (!NewStatePending())
         {
             FairMQParts payload;
             if (Receive(payload, fInChannelName) >= 0)
@@ -61,7 +61,7 @@ void FairMQProxy::Run()
     }
     else
     {
-        while (CheckCurrentState(RUNNING))
+        while (!NewStatePending())
         {
             unique_ptr<FairMQMessage> payload(fTransportFactory->CreateMessage());
             if (Receive(payload, fInChannelName) >= 0)

fairmq/devices/FairMQProxy.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/devices/FairMQSink.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**
@@ -57,7 +57,7 @@ class FairMQSink : public FairMQDevice//, public OutputPolicy
         LOG(info) << "Starting the benchmark and expecting to receive " << fMaxIterations << " messages.";
         auto tStart = std::chrono::high_resolution_clock::now();
 
-        while (CheckCurrentState(RUNNING))
+        while (!NewStatePending())
         {
             if (fMultipart)
             {

fairmq/devices/FairMQSplitter.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/devices/FairMQSplitter.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/nanomsg/FairMQPollerNN.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/nanomsg/FairMQPollerNN.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/nanomsg/FairMQSocketNN.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/nanomsg/FairMQSocketNN.h

@@ -1,8 +1,8 @@
 /********************************************************************************
  * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 

fairmq/ofi/CMakeLists.txt

@@ -1,12 +0,0 @@
-################################################################################
-#    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
-#                                                                              #
-#              This software is distributed under the terms of the             #
-#              GNU Lesser General Public Licence (LGPL) version 3,             #
-#                  copied verbatim in the file "LICENSE"                       #
-################################################################################
-
-protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS Control.proto)
-
-add_library(OfiTransport OBJECT ${PROTO_SRCS} ${PROTO_HDRS})
-target_include_directories(OfiTransport PRIVATE $<TARGET_PROPERTY:protobuf::libprotobuf,INTERFACE_INCLUDE_DIRECTORIES>)

fairmq/ofi/Context.cxx

@@ -10,21 +10,17 @@
 #include <fairmq/Tools.h>
 #include <FairMQLogger.h>
 
+#include <asiofi/version.hpp>
 #include <arpa/inet.h>
 #include <boost/version.hpp>
+#include <cassert>
 #include <cstring>
-#include <google/protobuf/stubs/common.h>
 #include <memory>
 #include <netinet/in.h>
-#include <rdma/fabric.h>
-#include <rdma/fi_domain.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_errno.h>
 #include <regex>
 #include <string>
 #include <string.h>
 #include <sys/socket.h>
-#include <zmq.h>
 
 namespace fair
 {
@@ -35,20 +31,14 @@ namespace ofi
 
 using namespace std;
 
-Context::Context(int numberIoThreads)
-    : fOfiDomain(nullptr)
-    , fOfiFabric(nullptr)
-    , fOfiInfo(nullptr)
-    , fOfiAddressVector(nullptr)
-    , fOfiEventQueue(nullptr)
-    , fZmqContext(zmq_ctx_new())
-    , fIoWork(fIoContext)
+Context::Context(FairMQTransportFactory& sendFactory,
+                 FairMQTransportFactory& receiveFactory,
+                 int numberIoThreads)
+    : fIoWork(fIoContext)
+    , fReceiveFactory(receiveFactory)
+    , fSendFactory(sendFactory)
+    , fSizeHint(2000000) // temporary hack to provide expected message size for receive
 {
-    if (!fZmqContext)
-        throw ContextError{tools::ToString("Failed creating zmq context, reason: ", zmq_strerror(errno))};
-
-    GOOGLE_PROTOBUF_VERIFY_VERSION;
-
     InitThreadPool(numberIoThreads);
 }
 
@@ -58,257 +48,27 @@ auto Context::InitThreadPool(int numberIoThreads) -> void
 
     for (int i = 1; i <= numberIoThreads; ++i) {
         fThreadPool.emplace_back([&, i, numberIoThreads]{
-            LOG(debug) << "I/O thread #" << i << "/" << numberIoThreads << " started";
+            LOG(debug) << "OFI transport: I/O thread #" << i << " of " << numberIoThreads << " started";
             fIoContext.run();
-            LOG(debug) << "I/O thread #" << i << "/" << numberIoThreads << " stopped";
+            LOG(debug) << "OFI transport: I/O thread #" << i << " of " << numberIoThreads << " stopped";
         });
     }
 }
 
-Context::~Context()
+auto Context::Reset() -> void
 {
     fIoContext.stop();
+}
+
+Context::~Context()
+{
     for (auto& thread : fThreadPool)
         thread.join();
-
-    if (zmq_ctx_term(fZmqContext) != 0)
-        LOG(error) << "Failed closing zmq context, reason: " << zmq_strerror(errno);
-
-	if (fOfiEventQueue) {
-        auto ret = fi_close(&fOfiEventQueue->fid);
-        if (ret != FI_SUCCESS)
-            LOG(error) << "Failed closing ofi event queue, reason: " << fi_strerror(ret);
-    }
-
-	if (fOfiAddressVector) {
-        auto ret = fi_close(&fOfiAddressVector->fid);
-        if (ret != FI_SUCCESS)
-            LOG(error) << "Failed closing ofi address vector, reason: " << fi_strerror(ret);
-    }
-
-	if (fOfiDomain) {
-        auto ret = fi_close(&fOfiDomain->fid);
-        if (ret != FI_SUCCESS)
-            LOG(error) << "Failed closing ofi domain, reason: " << fi_strerror(ret);
-    }
-
-	if (fOfiFabric) {
-        auto ret = fi_close(&fOfiFabric->fid);
-        if (ret != FI_SUCCESS)
-            LOG(error) << "Failed closing ofi fabric, reason: " << fi_strerror(ret);
-    }
 }
 
-auto Context::GetZmqVersion() const -> string
+auto Context::GetAsiofiVersion() const -> string
 {
-    int major, minor, patch;
-    zmq_version(&major, &minor, &patch);
-    return tools::ToString(major, ".", minor, ".", patch);
-}
-
-auto Context::GetOfiApiVersion() const -> string
-{
-    // Disable for now, does not compile with gcc 4.9.2 debian jessie
-    //auto ofi_version{fi_version()};
-    //return tools::ToString(FI_MAJOR(ofi_version), ".", FI_MINOR(ofi_version));
-    return "unknown";
-}
-
-auto Context::GetPbVersion() const -> string
-{
-    return google::protobuf::internal::VersionString(GOOGLE_PROTOBUF_VERSION);
-}
-
-auto Context::GetBoostVersion() const -> std::string
-{
-    return tools::ToString(BOOST_VERSION / 100000, ".", BOOST_VERSION / 100 % 1000, ".", BOOST_VERSION % 100);
-}
-
-auto Context::InitOfi(ConnectionType type, Address addr) -> void
-{
-    if (!fOfiInfo) {
-        sockaddr_in* sa = static_cast<sockaddr_in*>(malloc(sizeof(sockaddr_in)));
-        addr.Port = 0;
-        auto sa2 = ConvertAddress(addr);
-        memcpy(sa, &sa2, sizeof(sockaddr_in));
-
-        // Prepare fi_getinfo query
-        unique_ptr<fi_info, void(*)(fi_info*)> ofi_hints(fi_allocinfo(), fi_freeinfo);
-        ofi_hints->caps = FI_MSG;
-        //ofi_hints->mode = FI_CONTEXT;
-        ofi_hints->addr_format = FI_SOCKADDR_IN;
-        if (addr.Protocol == "tcp") {
-            ofi_hints->fabric_attr->prov_name = strdup("sockets");
-        } else if (addr.Protocol == "verbs") {
-            ofi_hints->fabric_attr->prov_name = strdup("verbs;ofi_rxm");
-        }
-        ofi_hints->ep_attr->type = FI_EP_RDM;
-        //ofi_hints->domain_attr->mr_mode = FI_MR_BASIC | FI_MR_SCALABLE;
-        ofi_hints->domain_attr->threading = FI_THREAD_SAFE;
-        ofi_hints->domain_attr->control_progress = FI_PROGRESS_AUTO;
-        ofi_hints->domain_attr->data_progress = FI_PROGRESS_AUTO;
-        ofi_hints->tx_attr->op_flags = FI_COMPLETION;
-        ofi_hints->rx_attr->op_flags = FI_COMPLETION;
-        if (type == ConnectionType::Bind) {
-            ofi_hints->src_addr = sa;
-            ofi_hints->src_addrlen = sizeof(sockaddr_in);
-            ofi_hints->dest_addr = nullptr;
-            ofi_hints->dest_addrlen = 0;
-        } else {
-            ofi_hints->src_addr = nullptr;
-            ofi_hints->src_addrlen = 0;
-            ofi_hints->dest_addr = sa;
-            ofi_hints->dest_addrlen = sizeof(sockaddr_in);
-        }
-
-        // Query fi_getinfo for fabric to use
-        auto res = fi_getinfo(FI_VERSION(1, 5), nullptr, nullptr, 0, ofi_hints.get(), &fOfiInfo);
-        if (res != 0) throw ContextError{tools::ToString("Failed querying fi_getinfo, reason: ", fi_strerror(res))};
-        if (!fOfiInfo) throw ContextError{"Could not find any ofi compatible fabric."};
-
-        // for(auto cursor{ofi_info}; cursor->next != nullptr; cursor = cursor->next) {
-            // LOG(debug) << fi_tostr(fOfiInfo, FI_TYPE_INFO);
-        // }
-        //
-    } else {
-        LOG(debug) << "Ofi info already queried. Skipping.";
-    }
-
-    OpenOfiFabric();
-//    OpenOfiEventQueue();
-    OpenOfiDomain();
-    OpenOfiAddressVector();
-}
-
-auto Context::OpenOfiFabric() -> void
-{
-    if (!fOfiFabric) {
-        assert(fOfiInfo);
-        fi_context ctx;
-        auto ret = fi_fabric(fOfiInfo->fabric_attr, &fOfiFabric, &ctx);
-        if (ret != FI_SUCCESS)
-            throw ContextError{tools::ToString("Failed opening ofi fabric, reason: ", fi_strerror(ret))};
-    } else {
-        // TODO Check, if requested fabric matches existing one.
-        // TODO Decide, if we want to support more than one fabric simultaneously.
-        LOG(debug) << "Ofi fabric already opened. Skipping.";
-    }
-}
-
-auto Context::OpenOfiDomain() -> void
-{
-    if (!fOfiDomain) {
-        assert(fOfiInfo);
-        assert(fOfiFabric);
-        fi_context ctx;
-        auto ret = fi_domain(fOfiFabric, fOfiInfo, &fOfiDomain, &ctx);
-        if (ret != FI_SUCCESS)
-            throw ContextError{tools::ToString("Failed opening ofi domain, reason: ", fi_strerror(ret))};
-    } else {
-        LOG(debug) << "Ofi domain already opened. Skipping.";
-    }
-}
-
-auto Context::OpenOfiEventQueue() -> void
-{
-    fi_eq_attr eqAttr = {100, 0, FI_WAIT_UNSPEC, 0, nullptr};
-    // size_t               size;      [> # entries for EQ <]
-    // uint64_t             flags;     [> operation flags <]
-    // enum fi_wait_obj     wait_obj;  [> requested wait object <]
-    // int                  signaling_vector; [> interrupt affinity <]
-    // struct fid_wait     *wait_set;  [> optional wait set <]
-    fi_context ctx;
-    auto ret = fi_eq_open(fOfiFabric, &eqAttr, &fOfiEventQueue, &ctx);
-    if (ret != FI_SUCCESS)
-        throw ContextError{tools::ToString("Failed opening ofi event queue, reason: ", fi_strerror(ret))};
-}
-
-auto Context::OpenOfiAddressVector() -> void
-{
-    if (!fOfiAddressVector) {
-        assert(fOfiDomain);
-        fi_av_attr attr = {fOfiInfo->domain_attr->av_type, 0, 1000, 0, nullptr, nullptr, 0};
-        // enum fi_av_type  type;        [> type of AV <]
-        // int              rx_ctx_bits; [> address bits to identify rx ctx <]
-        // size_t           count;       [> # entries for AV <]
-        // size_t           ep_per_node; [> # endpoints per fabric address <]
-        // const char       *name;       [> system name of AV <]
-        // void             *map_addr;   [> base mmap address <]
-        // uint64_t         flags;       [> operation flags <]
-        fi_context ctx;
-        auto ret = fi_av_open(fOfiDomain, &attr, &fOfiAddressVector, &ctx);
-        if (ret != FI_SUCCESS)
-            throw ContextError{tools::ToString("Failed opening ofi address vector, reason: ", fi_strerror(ret))};
-
-        //assert(fOfiEventQueue);
-        //ret = fi_av_bind(fOfiAddressVector, &fOfiEventQueue->fid, 0);
-        //if (ret != FI_SUCCESS)
-        //    throw ContextError{tools::ToString("Failed binding ofi event queue to address vector, reason: ", fi_strerror(ret))};
-    } else {
-        LOG(debug) << "Ofi address vector already opened. Skipping.";
-    }
-}
-
-auto Context::CreateOfiEndpoint() -> fid_ep*
-{
-    assert(fOfiDomain);
-    assert(fOfiInfo);
-    fid_ep* ep = nullptr;
-    fi_context ctx;
-    auto ret = fi_endpoint(fOfiDomain, fOfiInfo, &ep, &ctx);
-    if (ret != FI_SUCCESS)
-        throw ContextError{tools::ToString("Failed creating ofi endpoint, reason: ", fi_strerror(ret))};
-
-    //assert(fOfiEventQueue);
-    //ret = fi_ep_bind(ep, &fOfiEventQueue->fid, 0);
-    //if (ret != FI_SUCCESS)
-    //    throw ContextError{tools::ToString("Failed binding ofi event queue to ofi endpoint, reason: ", fi_strerror(ret))};
-
-    assert(fOfiAddressVector);
-    ret = fi_ep_bind(ep, &fOfiAddressVector->fid, 0);
-    if (ret != FI_SUCCESS)
-        throw ContextError{tools::ToString("Failed binding ofi address vector to ofi endpoint, reason: ", fi_strerror(ret))};
-
-    return ep;
-}
-
-auto Context::CreateOfiCompletionQueue(Direction dir) -> fid_cq*
-{
-    fid_cq* cq = nullptr;
-    fi_cq_attr attr = {0, 0, FI_CQ_FORMAT_DATA, FI_WAIT_UNSPEC, 0, FI_CQ_COND_NONE, nullptr};
-    if (dir == Direction::Receive) {
-        attr.size = fOfiInfo->rx_attr->size;
-    } else {
-        attr.size = fOfiInfo->tx_attr->size;
-    }
-	// size_t               size;      [> # entries for CQ <]
-	// uint64_t             flags;     [> operation flags <]
-	// enum fi_cq_format    format;    [> completion format <]
-	// enum fi_wait_obj     wait_obj;  [> requested wait object <]
-	// int                  signaling_vector; [> interrupt affinity <]
-	// enum fi_cq_wait_cond wait_cond; [> wait condition format <]
-	// struct fid_wait     *wait_set;  [> optional wait set <]
-    fi_context ctx;
-    auto ret = fi_cq_open(fOfiDomain, &attr, &cq, &ctx);
-    if (ret != FI_SUCCESS)
-        throw ContextError{tools::ToString("Failed creating ofi completion queue, reason: ", fi_strerror(ret))};
-    return cq;
-}
-
-auto Context::InsertAddressVector(sockaddr_in address) -> fi_addr_t
-{
-    fi_addr_t mappedAddress;
-    fi_context ctx;
-    auto ret = fi_av_insert(fOfiAddressVector, &address, 1, &mappedAddress, 0, &ctx);
-    if (ret != 1)
-        throw ContextError{tools::ToString("Failed to insert address into ofi address vector")};
-
-    return mappedAddress;
-}
-
-auto Context::AddressVectorLookup(fi_addr_t address) -> sockaddr_in
-{
-    throw ContextError("Not yet implemented");
+    return ASIOFI_VERSION;
 }
 
 auto Context::ConvertAddress(std::string address) -> Address
@@ -355,6 +115,16 @@ auto Context::VerifyAddress(const std::string& address) -> Address
     return addr;
 }
 
+auto Context::MakeReceiveMessage(size_t size) -> MessagePtr
+{
+    return fReceiveFactory.CreateMessage(size);
+}
+
+auto Context::MakeSendMessage(size_t size) -> MessagePtr
+{
+    return fSendFactory.CreateMessage(size);
+}
+
 } /* namespace ofi */
 } /* namespace mq */
 } /* namespace fair */

fairmq/ofi/Context.h

@@ -9,11 +9,16 @@
 #ifndef FAIR_MQ_OFI_CONTEXT_H
 #define FAIR_MQ_OFI_CONTEXT_H
 
-#include <boost/asio.hpp>
+#include <FairMQLogger.h>
+#include <FairMQTransportFactory.h>
+
+#include <asiofi/domain.hpp>
+#include <asiofi/fabric.hpp>
+#include <asiofi/info.hpp>
+#include <boost/asio/io_context.hpp>
 #include <memory>
 #include <netinet/in.h>
 #include <ostream>
-#include <rdma/fabric.h>
 #include <stdexcept>
 #include <string>
 #include <thread>
@@ -27,7 +32,20 @@ namespace ofi
 {
 
 enum class ConnectionType : bool { Bind, Connect };
-enum class Direction : bool { Receive, Transmit };
+
+struct Address {
+    std::string Protocol;
+    std::string Ip;
+    unsigned int Port;
+    friend auto operator<<(std::ostream& os, const Address& a) -> std::ostream&
+    {
+        return os << a.Protocol << "://" << a.Ip << ":" << a.Port;
+    }
+    friend auto operator==(const Address& lhs, const Address& rhs) -> bool
+    {
+        return (lhs.Protocol == rhs.Protocol) && (lhs.Ip == rhs.Ip) && (lhs.Port == rhs.Port);
+    }
+};
 
 /**
  * @class Context Context.h <fairmq/ofi/Context.h>
@@ -38,46 +56,34 @@ enum class Direction : bool { Receive, Transmit };
 class Context
 {
   public:
-    Context(int numberIoThreads = 2);
+    Context(FairMQTransportFactory& sendFactory,
+            FairMQTransportFactory& receiveFactory,
+            int numberIoThreads = 1);
     ~Context();
 
-    auto CreateOfiEndpoint() -> fid_ep*;
-    auto CreateOfiCompletionQueue(Direction dir) -> fid_cq*;
-    auto GetZmqVersion() const -> std::string;
-    auto GetOfiApiVersion() const -> std::string;
-    auto GetPbVersion() const -> std::string;
-    auto GetBoostVersion() const -> std::string;
-    auto GetZmqContext() const -> void* { return fZmqContext; }
-    auto GetIoContext() -> boost::asio::io_service& { return fIoContext; }
-    auto InsertAddressVector(sockaddr_in address) -> fi_addr_t;
-    auto AddressVectorLookup(fi_addr_t address) -> sockaddr_in;
-    struct Address {
-        std::string Protocol;
-        std::string Ip;
-        unsigned int Port;
-        friend auto operator<<(std::ostream& os, const Address& a) -> std::ostream& { return os << a.Protocol << "://" << a.Ip << ":" << a.Port; }
-    };
-    auto InitOfi(ConnectionType type, Address address) -> void;
+    auto GetAsiofiVersion() const -> std::string;
+    auto GetIoContext() -> boost::asio::io_context& { return fIoContext; }
     static auto ConvertAddress(std::string address) -> Address;
     static auto ConvertAddress(Address address) -> sockaddr_in;
     static auto ConvertAddress(sockaddr_in address) -> Address;
     static auto VerifyAddress(const std::string& address) -> Address;
+    auto Interrupt() -> void { LOG(debug) << "OFI transport: Interrupted (NOOP - not implemented)."; }
+    auto Resume() -> void { LOG(debug) << "OFI transport: Resumed (NOOP - not implemented)."; }
+    auto Reset() -> void;
+    auto MakeReceiveMessage(size_t size) -> MessagePtr;
+    auto MakeSendMessage(size_t size) -> MessagePtr;
+    size_t GetSizeHint() { return fSizeHint; } // temporary hack to provide expected message size for receive
+    void SetSizeHint(size_t size) { fSizeHint = size; } // temporary hack to provide expected message size for receive
 
   private:
-    void* fZmqContext;
-    fi_info* fOfiInfo;
-    fid_fabric* fOfiFabric;
-    fid_domain* fOfiDomain;
-    fid_av* fOfiAddressVector;
-    fid_eq* fOfiEventQueue;
-    boost::asio::io_service fIoContext;
-    boost::asio::io_service::work fIoWork;
+    boost::asio::io_context fIoContext;
+    boost::asio::io_context::work fIoWork;
     std::vector<std::thread> fThreadPool;
+    FairMQTransportFactory& fReceiveFactory;
+    FairMQTransportFactory& fSendFactory;
+
+    size_t fSizeHint; // temporary hack to provide expected message size for receive
 
-    auto OpenOfiFabric() -> void;
-    auto OpenOfiEventQueue() -> void;
-    auto OpenOfiDomain() -> void;
-    auto OpenOfiAddressVector() -> void;
     auto InitThreadPool(int numberIoThreads) -> void;
 }; /* class Context */
 

fairmq/ofi/Control.proto

@@ -1,25 +0,0 @@
-syntax = "proto3";
-option optimize_for = SPEED;
-
-package fair.mq.ofi;
-
-message DataAddressAnnouncement {
-  uint32 ipv4 = 1; // in_addr_t from <netinet/in.h>
-  uint32 port = 2; // in_port_t from <netinet/in.h>
-}
-
-message PostBuffer {
-  uint64 size = 1; // buffer size (size_t)
-}
-
-message PostBufferAcknowledgement {
-  uint64 size = 1; // size_t
-}
-
-message ControlMessage {
-  oneof type {
-    DataAddressAnnouncement data_address_announcement = 1;
-    PostBuffer post_buffer = 2;
-    PostBufferAcknowledgement post_buffer_acknowledgement = 3;
-  }    
-}

fairmq/ofi/ControlMessages.h

@@ -0,0 +1,99 @@
+/********************************************************************************
+ *    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_OFI_CONTROLMESSAGES_H
+#define FAIR_MQ_OFI_CONTROLMESSAGES_H
+
+#include <FairMQLogger.h>
+#include <boost/asio/buffer.hpp>
+#include <boost/container/pmr/memory_resource.hpp>
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <type_traits>
+
+namespace boost {
+namespace asio {
+
+template<typename PodType>
+auto buffer(const PodType& obj) -> boost::asio::const_buffer
+{
+    return boost::asio::const_buffer(static_cast<const void*>(&obj), sizeof(PodType));
+}
+
+}   // namespace asio
+}   // namespace boost
+
+namespace fair {
+namespace mq {
+namespace ofi {
+
+enum class ControlMessageType
+{
+    DataAddressAnnouncement = 1,
+    PostBuffer,
+    PostBufferAcknowledgement
+};
+
+struct ControlMessage
+{
+    ControlMessageType type;
+};
+
+struct DataAddressAnnouncement : ControlMessage
+{
+    uint32_t ipv4;   // in_addr_t from <netinet/in.h>
+    uint32_t port;   // in_port_t from <netinet/in.h>
+};
+
+struct PostBuffer : ControlMessage
+{
+    uint64_t size;   // buffer size (size_t)
+};
+
+template<typename T>
+using unique_ptr = std::unique_ptr<T, std::function<void(T*)>>;
+
+template<typename T, typename... Args>
+auto MakeControlMessageWithPmr(boost::container::pmr::memory_resource* pmr, Args&&... args)
+    -> ofi::unique_ptr<T>
+{
+    void* mem = pmr->allocate(sizeof(T));
+    T* ctrl = new (mem) T(std::forward<Args>(args)...);
+
+    if (std::is_same<T, DataAddressAnnouncement>::value) {
+        ctrl->type = ControlMessageType::DataAddressAnnouncement;
+    } else if (std::is_same<T, PostBuffer>::value) {
+        ctrl->type = ControlMessageType::PostBuffer;
+    }
+
+    return ofi::unique_ptr<T>(ctrl, [=](T* p) {
+        p->~T();
+        pmr->deallocate(p, sizeof(T));
+    });
+}
+
+template<typename T, typename... Args>
+auto MakeControlMessage(Args&&... args) -> T
+{
+    T ctrl = T(std::forward<Args>(args)...);
+
+    if (std::is_same<T, DataAddressAnnouncement>::value) {
+        ctrl.type = ControlMessageType::DataAddressAnnouncement;
+    } else if (std::is_same<T, PostBuffer>::value) {
+        ctrl.type = ControlMessageType::PostBuffer;
+    }
+
+    return ctrl;
+}
+
+}   // namespace ofi
+}   // namespace mq
+}   // namespace fair
+
+#endif /* FAIR_MQ_OFI_CONTROLMESSAGES_H */ 

fairmq/ofi/Message.cxx

@@ -10,6 +10,7 @@
 #include <fairmq/Tools.h>
 #include <FairMQLogger.h>
 
+#include <asiofi.hpp>
 #include <cassert>
 #include <cstdlib>
 #include <zmq.h>
@@ -23,38 +24,50 @@ namespace ofi
 
 using namespace std;
 
-Message::Message()
+Message::Message(boost::container::pmr::memory_resource* pmr)
     : fInitialSize(0)
     , fSize(0)
     , fData(nullptr)
     , fFreeFunction(nullptr)
     , fHint(nullptr)
+    , fPmr(pmr)
 {
 }
 
-Message::Message(const size_t size)
+Message::Message(boost::container::pmr::memory_resource* pmr, const size_t size)
     : fInitialSize(size)
     , fSize(size)
     , fData(nullptr)
     , fFreeFunction(nullptr)
     , fHint(nullptr)
+    , fPmr(pmr)
 {
     if (size) {
-        fData = malloc(size);
+        // static void* buffer = fPmr->allocate(size);
+        // fData = buffer;
+        fData = fPmr->allocate(size);
         assert(fData);
     }
 }
 
-Message::Message(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
+Message::Message(boost::container::pmr::memory_resource* pmr,
+                 void* data,
+                 const size_t size,
+                 fairmq_free_fn* ffn,
+                 void* hint)
     : fInitialSize(size)
     , fSize(size)
     , fData(data)
     , fFreeFunction(ffn)
     , fHint(hint)
-{
-}
+    , fPmr(pmr)
+{}
 
-Message::Message(FairMQUnmanagedRegionPtr& /*region*/, void* /*data*/, const size_t /*size*/, void* /*hint*/)
+Message::Message(boost::container::pmr::memory_resource* /*pmr*/,
+                 FairMQUnmanagedRegionPtr& /*region*/,
+                 void* /*data*/,
+                 const size_t /*size*/,
+                 void* /*hint*/)
 {
     throw MessageError{"Not yet implemented."};
 }
@@ -62,9 +75,11 @@ Message::Message(FairMQUnmanagedRegionPtr& /*region*/, void* /*data*/, const siz
 auto Message::Rebuild() -> void
 {
     if (fFreeFunction) {
-      fFreeFunction(fData, fHint);
+        fFreeFunction(fData, fHint);
     } else {
-      free(fData);
+        if (fData) {
+            fPmr->deallocate(fData, fSize);
+        }
     }
     fData = nullptr;
     fInitialSize = 0;
@@ -78,10 +93,12 @@ auto Message::Rebuild(const size_t size) -> void
     if (fFreeFunction) {
       fFreeFunction(fData, fHint);
     } else {
-      free(fData);
+        if (fData) {
+            fPmr->deallocate(fData, fSize);
+        }
     }
     if (size) {
-        fData = malloc(size);
+        fData = fPmr->allocate(size);
         assert(fData);
     } else {
         fData = nullptr;
@@ -97,10 +114,12 @@ auto Message::Rebuild(void* /*data*/, const size_t size, fairmq_free_fn* ffn, vo
     if (fFreeFunction) {
       fFreeFunction(fData, fHint);
     } else {
-      free(fData);
+        if (fData) {
+            fPmr->deallocate(fData, fSize);
+        }
     }
     if (size) {
-        fData = malloc(size);
+        fData = fPmr->allocate(size);
         assert(fData);
     } else {
         fData = nullptr;
@@ -141,9 +160,11 @@ auto Message::Copy(const fair::mq::Message& /*msg*/) -> void
 Message::~Message()
 {
     if (fFreeFunction) {
-      fFreeFunction(fData, fHint);
+        fFreeFunction(fData, fHint);
     } else {
-      free(fData);
+        if (fData) {
+            fPmr->deallocate(fData, fSize);
+        }
     }
 }
 

fairmq/ofi/Message.h

@@ -12,10 +12,10 @@
 #include <FairMQMessage.h>
 #include <FairMQUnmanagedRegion.h>
 
-#include <zmq.h>
-
-#include <cstddef> // size_t
+#include <asiofi.hpp>
 #include <atomic>
+#include <cstddef> // size_t
+#include <zmq.h>
 
 namespace fair
 {
@@ -33,10 +33,18 @@ namespace ofi
 class Message final : public fair::mq::Message
 {
   public:
-    Message();
-    Message(const size_t size);
-    Message(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr);
-    Message(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0);
+    Message(boost::container::pmr::memory_resource* pmr);
+    Message(boost::container::pmr::memory_resource* pmr, const size_t size);
+    Message(boost::container::pmr::memory_resource* pmr,
+            void* data,
+            const size_t size,
+            fairmq_free_fn* ffn,
+            void* hint = nullptr);
+    Message(boost::container::pmr::memory_resource* pmr,
+            FairMQUnmanagedRegionPtr& region,
+            void* data,
+            const size_t size,
+            void* hint = 0);
 
     Message(const Message&) = delete;
     Message operator=(const Message&) = delete;
@@ -62,6 +70,7 @@ class Message final : public fair::mq::Message
     void* fData;
     fairmq_free_fn* fFreeFunction;
     void* fHint;
+    boost::container::pmr::memory_resource* fPmr;
 }; /* class Message */  
 
 } /* namespace ofi */

fairmq/ofi/Poller.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
@@ -28,13 +28,13 @@ Poller::Poller(const vector<FairMQChannel>& channels)
     fItems = new zmq_pollitem_t[fNumItems];
 
     for (int i = 0; i < fNumItems; ++i) {
-        fItems[i].socket = static_cast<const Socket*>(&(channels.at(i).GetSocket()))->GetSocket();
+        fItems[i].socket = static_cast<Socket*>(&(channels.at(i).GetSocket()))->GetSocket();
         fItems[i].fd = 0;
         fItems[i].revents = 0;
 
         int type = 0;
         size_t size = sizeof(type);
-        zmq_getsockopt(static_cast<const Socket*>(&(channels.at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+        zmq_getsockopt(static_cast<Socket*>(&(channels.at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
 
         SetItemEvents(fItems[i], type);
     }
@@ -46,13 +46,13 @@ Poller::Poller(const vector<const FairMQChannel*>& channels)
     fItems = new zmq_pollitem_t[fNumItems];
 
     for (int i = 0; i < fNumItems; ++i) {
-        fItems[i].socket = static_cast<const Socket*>(&(channels.at(i)->GetSocket()))->GetSocket();
+        fItems[i].socket = static_cast<Socket*>(&(channels.at(i)->GetSocket()))->GetSocket();
         fItems[i].fd = 0;
         fItems[i].revents = 0;
 
         int type = 0;
         size_t size = sizeof(type);
-        zmq_getsockopt(static_cast<const Socket*>(&(channels.at(i)->GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+        zmq_getsockopt(static_cast<Socket*>(&(channels.at(i)->GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
 
         SetItemEvents(fItems[i], type);
     }
@@ -76,13 +76,13 @@ Poller::Poller(const unordered_map<string, vector<FairMQChannel>>& channelsMap,
             for (unsigned int i = 0; i < channelsMap.at(channel).size(); ++i) {
                 index = fOffsetMap[channel] + i;
 
-                fItems[index].socket = static_cast<const Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket();
+                fItems[index].socket = static_cast<Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket();
                 fItems[index].fd = 0;
                 fItems[index].revents = 0;
 
                 int type = 0;
                 size_t size = sizeof(type);
-                zmq_getsockopt(static_cast<const Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
+                zmq_getsockopt(static_cast<Socket*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket(), ZMQ_TYPE, &type, &size);
 
                 SetItemEvents(fItems[index], type);
             }
@@ -124,7 +124,7 @@ auto Poller::CheckOutput(const int index) -> bool
     return fItems[index].revents & ZMQ_POLLOUT;
 }
 
-auto Poller::CheckInput(const string channelKey, const int index) -> bool
+auto Poller::CheckInput(const string& channelKey, const int index) -> bool
 {
     try {
         return fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLIN;
@@ -136,7 +136,7 @@ auto Poller::CheckInput(const string channelKey, const int index) -> bool
     }
 }
 
-auto Poller::CheckOutput(const string channelKey, const int index) -> bool
+auto Poller::CheckOutput(const string& channelKey, const int index) -> bool
 {
     try {
         return fItems[fOffsetMap.at(channelKey) + index].revents & ZMQ_POLLOUT;

fairmq/ofi/Poller.h

@@ -51,8 +51,8 @@ class Poller final : public FairMQPoller
     auto Poll(const int timeout) -> void override;
     auto CheckInput(const int index) -> bool override;
     auto CheckOutput(const int index) -> bool override;
-    auto CheckInput(const std::string channelKey, const int index) -> bool override;
-    auto CheckOutput(const std::string channelKey, const int index) -> bool override;
+    auto CheckInput(const std::string& channelKey, const int index) -> bool override;
+    auto CheckOutput(const std::string& channelKey, const int index) -> bool override;
 
     ~Poller() override;
 

fairmq/ofi/Socket.cxx

@@ -6,21 +6,27 @@
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
+#include <fairmq/ofi/ControlMessages.h>
 #include <fairmq/ofi/Socket.h>
 #include <fairmq/ofi/TransportFactory.h>
 #include <fairmq/Tools.h>
 #include <FairMQLogger.h>
 
-#include <arpa/inet.h>
+#include <asiofi.hpp>
+#include <azmq/message.hpp>
+#include <boost/asio/buffer.hpp>
+#include <boost/asio/post.hpp>
+#include <chrono>
 #include <cstring>
-#include <netinet/in.h>
-#include <rdma/fabric.h>
-#include <rdma/fi_endpoint.h>
-#include <rdma/fi_cm.h>
+#include <functional>
+#include <memory>
 #include <sstream>
 #include <string.h>
 #include <sys/socket.h>
-#include <zmq.h>
+#include <thread>
+
+#include <mutex>
+#include <condition_variable>
 
 namespace fair
 {
@@ -31,672 +37,394 @@ namespace ofi
 
 using namespace std;
 
-Socket::Socket(Context& context, const string& type, const string& name, const string& id /*= ""*/, FairMQTransportFactory* fac)
-    : FairMQSocket{fac}
+Socket::Socket(Context& context, const string& type, const string& name, const string& id /*= ""*/)
+    : fContext(context)
+    , fOfiInfo(nullptr)
+    , fOfiFabric(nullptr)
+    , fOfiDomain(nullptr)
+    , fPassiveEndpoint(nullptr)
     , fDataEndpoint(nullptr)
-    , fDataCompletionQueueTx(nullptr)
-    , fDataCompletionQueueRx(nullptr)
     , fId(id + "." + name + "." + type)
-    , fControlSocket(nullptr)
-    , fMonitorSocket(nullptr)
-    , fSndTimeout(100)
-    , fRcvTimeout(100)
-    , fContext(context)
-    , fWaitingForControlPeer(false)
-    , fIoStrand(fContext.GetIoContext())
     , fBytesTx(0)
     , fBytesRx(0)
     , fMessagesTx(0)
     , fMessagesRx(0)
+    , fSndTimeout(100)
+    , fRcvTimeout(100)
+    , fRecvQueueWrite(fContext.GetIoContext(), ZMQ_PUSH)
+    , fRecvQueueRead(fContext.GetIoContext(), ZMQ_PULL)
+    , fSendSem(fContext.GetIoContext(), 300)
+    , fRecvSem(fContext.GetIoContext(), 300)
+    , fNeedOfiMemoryRegistration(false)
+    , fBound(false)
+    , fConnected(false)
 {
     if (type != "pair") {
         throw SocketError{tools::ToString("Socket type '", type, "' not implemented for ofi transport.")};
     } else {
-        fControlSocket = zmq_socket(fContext.GetZmqContext(), ZMQ_PAIR);
+        // TODO wire this up with config
+        azmq::socket::snd_hwm send_max(300);
+        azmq::socket::rcv_hwm recv_max(300);
+        fRecvQueueRead.set_option(send_max);
+        fRecvQueueRead.set_option(recv_max);
+        fRecvQueueWrite.set_option(send_max);
+        fRecvQueueWrite.set_option(recv_max);
 
-        if (fControlSocket == nullptr)
-            throw SocketError{tools::ToString("Failed creating zmq meta socket ", fId, ", reason: ", zmq_strerror(errno))};
-
-        if (zmq_setsockopt(fControlSocket, ZMQ_IDENTITY, fId.c_str(), fId.length()) != 0)
-            throw SocketError{tools::ToString("Failed setting ZMQ_IDENTITY socket option, reason: ", zmq_strerror(errno))};
-
-        // Tell socket to try and send/receive outstanding messages for <linger> milliseconds before terminating.
-        // Default value for ZeroMQ is -1, which is to wait forever.
-        int linger = 1000;
-        if (zmq_setsockopt(fControlSocket, ZMQ_LINGER, &linger, sizeof(linger)) != 0)
-            throw SocketError{tools::ToString("Failed setting ZMQ_LINGER socket option, reason: ", zmq_strerror(errno))};
-
-        // TODO enable again and implement retries
-        // if (zmq_setsockopt(fControlSocket, ZMQ_SNDTIMEO, &fSndTimeout, sizeof(fSndTimeout)) != 0)
-        //     throw SocketError{tools::ToString("Failed setting ZMQ_SNDTIMEO socket option, reason: ", zmq_strerror(errno))};
-        //
-        // if (zmq_setsockopt(fControlSocket, ZMQ_RCVTIMEO, &fRcvTimeout, sizeof(fRcvTimeout)) != 0)
-        //     throw SocketError{tools::ToString("Failed setting ZMQ_RCVTIMEO socket option, reason: ", zmq_strerror(errno))};
-
-        fMonitorSocket = zmq_socket(fContext.GetZmqContext(), ZMQ_PAIR);
-
-        if (fMonitorSocket == nullptr)
-            throw SocketError{tools::ToString("Failed creating zmq monitor socket ", fId, ", reason: ", zmq_strerror(errno))};
-
-        auto mon_addr = tools::ToString("inproc://", fId);
-        if (zmq_socket_monitor(fControlSocket, mon_addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_CONNECTED) < 0)
-            throw SocketError{tools::ToString("Failed setting up monitor on meta socket, reason: ", zmq_strerror(errno))};
-
-        if (zmq_connect(fMonitorSocket, mon_addr.c_str()) != 0)
-            throw SocketError{tools::ToString("Failed connecting monitor socket to meta socket, reason: ", zmq_strerror(errno))};
+        // Setup internal queue
+        auto hashed_id = hash<string>()(fId);
+        auto queue_id = tools::ToString("inproc://TXQUEUE", hashed_id);
+        queue_id = tools::ToString("inproc://RXQUEUE", hashed_id);
+        LOG(debug) << "OFI transport (" << fId << "): " << "Binding RQR: " << queue_id;
+        fRecvQueueRead.bind(queue_id);
+        LOG(debug) << "OFI transport (" << fId << "): " << "Connecting RQW: " << queue_id;
+        fRecvQueueWrite.connect(queue_id);
     }
 }
 
-auto Socket::Bind(const string& address) -> bool
-try {
-    auto addr = Context::VerifyAddress(address);
-    BindControlSocket(addr);
-    fContext.InitOfi(ConnectionType::Bind, addr);
-    InitDataEndpoint();
-    fWaitingForControlPeer = true;
-    return true;
-}
-catch (const SilentSocketError& e)
+auto Socket::InitOfi(Address addr) -> void
 {
+    if (!fOfiInfo) {
+      assert(!fOfiFabric);
+      assert(!fOfiDomain);
+
+      asiofi::hints hints;
+      if (addr.Protocol == "tcp") {
+          hints.set_provider("sockets");
+      } else if (addr.Protocol == "verbs") {
+          hints.set_provider("verbs");
+      }
+      if (fRemoteAddr == addr) {
+          fOfiInfo = tools::make_unique<asiofi::info>(addr.Ip.c_str(), to_string(addr.Port).c_str(), 0, hints);
+      } else {
+          fOfiInfo = tools::make_unique<asiofi::info>(addr.Ip.c_str(), to_string(addr.Port).c_str(), FI_SOURCE, hints);
+      }
+
+      LOG(debug) << "OFI transport: " << *fOfiInfo;
+
+      fOfiFabric = tools::make_unique<asiofi::fabric>(*fOfiInfo);
+
+      fOfiDomain = tools::make_unique<asiofi::domain>(*fOfiFabric);
+    }
+}
+
+auto Socket::Bind(const string& addr) -> bool
+try {
+    fBound = false;
+    fLocalAddr = Context::VerifyAddress(addr);
+    if (fLocalAddr.Protocol == "verbs") {
+        fNeedOfiMemoryRegistration = true;
+    }
+
+    InitOfi(fLocalAddr);
+
+    fPassiveEndpoint = tools::make_unique<asiofi::passive_endpoint>(fContext.GetIoContext(), *fOfiFabric);
+    //fPassiveEndpoint->set_local_address(Context::ConvertAddress(fLocalAddr));
+
+    assert(!fDataEndpoint);
+
+    fPassiveEndpoint->listen([&](asiofi::info&& info) {
+        LOG(debug) << "OFI transport (" << fId << "): data band connection request received. Accepting ...";
+        fDataEndpoint = tools::make_unique<asiofi::connected_endpoint>(fContext.GetIoContext(), *fOfiDomain, info);
+        fDataEndpoint->enable();
+        fDataEndpoint->accept([&]() {
+            LOG(debug) << "OFI transport (" << fId << "): data band connection accepted.";
+
+            boost::asio::post(fContext.GetIoContext(), bind(&Socket::RecvQueueReader, this));
+            fBound = true;
+        });
+    });
+
+    LOG(debug) << "OFI transport (" << fId << "): data band bound to " << fLocalAddr;
+
+    while (!fBound) {
+        this_thread::sleep_for(chrono::milliseconds(100));
+    }
+
+    return true;
+} catch (const SilentSocketError& e) {// TODO catch the correct ofi error
     // do not print error in this case, this is handled by FairMQDevice
     // in case no connection could be established after trying a number of random ports from a range.
     return false;
-}
-catch (const SocketError& e)
-{
-    LOG(error) << e.what();
+} catch (const SocketError& e) {
+    LOG(error) << "OFI transport: " << e.what();
     return false;
 }
 
 auto Socket::Connect(const string& address) -> bool
-{
-    auto addr = Context::VerifyAddress(address);
-    ConnectControlSocket(addr);
-    fContext.InitOfi(ConnectionType::Connect, addr);
-    InitDataEndpoint();
-    fWaitingForControlPeer = true;
-    return true;
-}
-
-auto Socket::BindControlSocket(Context::Address address) -> void
-{
-    auto addr = tools::ToString("tcp://", address.Ip, ":", address.Port);
-
-    if (zmq_bind(fControlSocket, addr.c_str()) != 0) {
-        if (errno == EADDRINUSE) throw SilentSocketError("EADDRINUSE");
-        throw SocketError(tools::ToString("Failed binding control socket ", fId, ", reason: ", zmq_strerror(errno)));
+try {
+    fConnected = false;
+    fRemoteAddr = Context::VerifyAddress(address);
+    if (fRemoteAddr.Protocol == "verbs") {
+        fNeedOfiMemoryRegistration = true;
     }
-}
 
-auto Socket::ConnectControlSocket(Context::Address address) -> void
-{
-    auto addr = tools::ToString("tcp://", address.Ip, ":", address.Port);
+    InitOfi(fRemoteAddr);
 
-    if (zmq_connect(fControlSocket, addr.c_str()) != 0)
-        throw SocketError(tools::ToString("Failed connecting control socket ", fId, ", reason: ", zmq_strerror(errno)));
-}
+    assert(!fDataEndpoint);
 
-auto Socket::ProcessDataAddressAnnouncement(std::unique_ptr<ControlMessage> ctrl) -> void
-{
-    assert(ctrl->has_data_address_announcement());
-    auto daa = ctrl->data_address_announcement();
+    fDataEndpoint = tools::make_unique<asiofi::connected_endpoint>(fContext.GetIoContext(), *fOfiDomain);
+    fDataEndpoint->enable();
 
-    sockaddr_in remoteAddr;
-    remoteAddr.sin_family = AF_INET;
-    remoteAddr.sin_port = daa.port();
-    remoteAddr.sin_addr.s_addr = daa.ipv4();
+    LOG(debug) << "OFI transport (" << fId << "): Sending data band connection request to " << fRemoteAddr;
 
-    LOG(debug) << "Data address announcement of remote ofi endpoint received: " << Context::ConvertAddress(remoteAddr);
-    fRemoteDataAddr = fContext.InsertAddressVector(remoteAddr);
-}
-
-auto Socket::InitDataEndpoint() -> void
-{
-    if (!fDataEndpoint) {
-        try {
-            fDataEndpoint = fContext.CreateOfiEndpoint();
-        } catch (ContextError& e) {
-            throw SocketError(tools::ToString("Failed creating ofi endpoint, reason: ", e.what()));
+    fDataEndpoint->connect(Context::ConvertAddress(fRemoteAddr), [&](asiofi::eq::event event) {
+        LOG(debug) << "OFI transport (" << fId << "): data band conn event happened";
+        if (event == asiofi::eq::event::connected) {
+            LOG(debug) << "OFI transport (" << fId << "): data band connected.";
+            boost::asio::post(fContext.GetIoContext(), bind(&Socket::RecvQueueReader, this));
+            fConnected = true;
+        } else {
+            LOG(error) << "Could not connect on the first try";
         }
+    });
 
-        if (!fDataCompletionQueueTx)
-            fDataCompletionQueueTx = fContext.CreateOfiCompletionQueue(Direction::Transmit);
-        auto ret = fi_ep_bind(fDataEndpoint, &fDataCompletionQueueTx->fid, FI_TRANSMIT);
-        if (ret != FI_SUCCESS)
-            throw SocketError(tools::ToString("Failed binding ofi transmit completion queue to endpoint, reason: ", fi_strerror(ret)));
-
-        if (!fDataCompletionQueueRx)
-            fDataCompletionQueueRx = fContext.CreateOfiCompletionQueue(Direction::Receive);
-        ret = fi_ep_bind(fDataEndpoint, &fDataCompletionQueueRx->fid, FI_RECV);
-        if (ret != FI_SUCCESS)
-            throw SocketError(tools::ToString("Failed binding ofi receive completion queue to endpoint, reason: ", fi_strerror(ret)));
-
-        ret = fi_enable(fDataEndpoint);
-        if (ret != FI_SUCCESS)
-            throw SocketError(tools::ToString("Failed enabling ofi endpoint, reason: ", fi_strerror(ret)));
+    while (!fConnected) {
+        this_thread::sleep_for(chrono::milliseconds(100));
     }
+
+    return true;
+} catch (const SilentSocketError& e) {
+    // do not print error in this case, this is handled by FairMQDevice
+    return false;
+} catch (const exception& e) {
+    LOG(error) << "OFI transport: " << e.what();
+    return false;
 }
 
-void free_string(void* /*data*/, void* hint)
+auto Socket::Send(MessagePtr& msg, const int /*timeout*/) -> int
 {
-    delete static_cast<string*>(hint);
+    // LOG(debug) << "OFI transport (" << fId << "): ENTER Send: data=" << msg->GetData() << ",size=" << msg->GetSize();
+
+    try {
+        fSendSem.wait();
+        size_t size = msg->GetSize();
+        OnSend(msg);
+        return size;
+    } catch (const exception& e) {
+        LOG(error) << e.what();
+        return -1;
+    } catch (const boost::system::error_code& e) {
+        LOG(error) << e;
+        return -1;
+    }
 }
 
-auto Socket::AnnounceDataAddress() -> void
-try {
-    using namespace google::protobuf;
-
-    size_t addrlen = sizeof(sockaddr_in);
-    auto ret = fi_getname(&fDataEndpoint->fid, &fLocalDataAddr, &addrlen);
-    if (ret != FI_SUCCESS)
-        throw SocketError(tools::ToString("Failed retrieving native address from ofi endpoint, reason: ", fi_strerror(ret)));
-    assert(addrlen == sizeof(sockaddr_in));
-
-    LOG(debug) << "Address of local ofi endpoint in socket " << fId << ": " << Context::ConvertAddress(fLocalDataAddr);
-
-    // Create new control message
-    auto ctrl = tools::make_unique<ControlMessage>();
-    auto daa = tools::make_unique<DataAddressAnnouncement>();
-
-    // Fill data address announcement
-    daa->set_ipv4(fLocalDataAddr.sin_addr.s_addr);
-    daa->set_port(fLocalDataAddr.sin_port);
-
-    // Fill control message
-    ctrl->set_allocated_data_address_announcement(daa.release());
-    assert(ctrl->IsInitialized());
-
-    SendControlMessage(move(ctrl));
-} catch (const SocketError& e) {
-    throw SocketError(tools::ToString("Failed to announce data address, reason: ", e.what()));
-}
-
-auto Socket::SendControlMessage(unique_ptr<ControlMessage> ctrl) -> void
+auto Socket::OnSend(MessagePtr& msg) -> void
 {
-    assert(fControlSocket);
-    // LOG(debug) << "About to send control message: " << ctrl->DebugString();
-
-    // Serialize
-    string* str = new string();
-    ctrl->SerializeToString(str);
-    zmq_msg_t msg;
-    auto ret = zmq_msg_init_data(&msg, const_cast<char*>(str->c_str()), str->length(), free_string, str);
-    assert(ret == 0);
-
-    // Send
-    if (zmq_msg_send(&msg, fControlSocket, 0) == -1) {
-        zmq_msg_close(&msg);
-        throw SocketError(tools::ToString("Failed to send control message, reason: ", zmq_strerror(errno)));
-    }
-}
-
-auto Socket::ReceiveControlMessage() -> unique_ptr<ControlMessage>
-{
-    assert(fControlSocket);
-
-    // Receive 
-    zmq_msg_t msg;
-    auto ret = zmq_msg_init(&msg);
-    assert(ret == 0);
-    if (zmq_msg_recv(&msg, fControlSocket, 0) == -1) {
-        zmq_msg_close(&msg);
-        throw SocketError(tools::ToString("Failed to receive control message, reason: ", zmq_strerror(errno)));
-    }
-
-    // Deserialize
-    auto ctrl = tools::make_unique<ControlMessage>();
-    ctrl->ParseFromArray(zmq_msg_data(&msg), zmq_msg_size(&msg));
-    
-    zmq_msg_close(&msg);
-    // LOG(debug) << "Received control message: " << ctrl->DebugString();
-    return ctrl;
-}
-
-auto Socket::WaitForControlPeer() -> void
-{
-    assert(fWaitingForControlPeer);
-
-    // First frame in message contains event number and value
-    zmq_msg_t msg;
-    zmq_msg_init(&msg);
-    if (zmq_msg_recv(&msg, fMonitorSocket, 0) == -1)
-        throw SocketError(tools::ToString("Failed to get monitor event, reason: ", zmq_strerror(errno)));
-
-    uint8_t* data = (uint8_t*) zmq_msg_data(&msg);
-    uint16_t event = *(uint16_t*)(data);
-    int value = *(uint32_t *)(data + 2);
-
-    // Second frame in message contains event address
-    zmq_msg_init(&msg);
-    if (zmq_msg_recv(&msg, fMonitorSocket, 0) == -1)
-        throw SocketError(tools::ToString("Failed to get monitor event, reason: ", zmq_strerror(errno)));
-
-    if (event == ZMQ_EVENT_ACCEPTED) {
-        // string localAddress = string(static_cast<char*>(zmq_msg_data(&msg)), zmq_msg_size(&msg));
-        sockaddr_in remoteAddr;
-        socklen_t addrSize = sizeof(sockaddr_in);
-        int ret = getpeername(value, (sockaddr*)&remoteAddr, &addrSize);
-        if (ret != 0)
-            throw SocketError(tools::ToString("Failed retrieving remote address, reason: ", strerror(errno)));
-        string remoteIp(inet_ntoa(remoteAddr.sin_addr));
-        int remotePort = ntohs(remoteAddr.sin_port);
-        LOG(debug) << "Accepted control peer connection from " << remoteIp << ":" << remotePort;
-    } else if (event == ZMQ_EVENT_CONNECTED) {
-        LOG(debug) << "Connected successfully to control peer";
-    } else {
-        LOG(debug) << "Unknown monitor event received: " << event << ". Ignoring.";
-    }
-
-    fWaitingForControlPeer = false;
-}
-
-auto Socket::Send(MessagePtr& msg, const int timeout) -> int { return SendImpl(msg, 0, timeout); }
-auto Socket::Receive(MessagePtr& msg, const int timeout) -> int { return ReceiveImpl(msg, 0, timeout); }
-auto Socket::Send(std::vector<MessagePtr>& msgVec, const int timeout) -> int64_t { return SendImpl(msgVec, 0, timeout); }
-auto Socket::Receive(std::vector<MessagePtr>& msgVec, const int timeout) -> int64_t { return ReceiveImpl(msgVec, 0, timeout); }
-
-auto Socket::TrySend(MessagePtr& msg) -> int { return SendImpl(msg, ZMQ_DONTWAIT, 0); }
-auto Socket::TryReceive(MessagePtr& msg) -> int { return ReceiveImpl(msg, ZMQ_DONTWAIT, 0); }
-auto Socket::TrySend(std::vector<MessagePtr>& msgVec) -> int64_t { return SendImpl(msgVec, ZMQ_DONTWAIT, 0); }
-auto Socket::TryReceive(std::vector<MessagePtr>& msgVec) -> int64_t { return ReceiveImpl(msgVec, ZMQ_DONTWAIT, 0); }
-
-auto Socket::SendImpl(FairMQMessagePtr& msg, const int flags, const int timeout) -> int
-try {
-    if (fWaitingForControlPeer) {
-        WaitForControlPeer(); 
-        AnnounceDataAddress();
-        ProcessDataAddressAnnouncement(ReceiveControlMessage());
-    }
+    // LOG(debug) << "OFI transport (" << fId << "): ENTER OnSend";
 
     auto size = msg->GetSize();
 
-    // Create and send control message
-    auto ctrl = tools::make_unique<ControlMessage>();
-    auto buf = tools::make_unique<PostBuffer>();
-    buf->set_size(size);
-    ctrl->set_allocated_post_buffer(buf.release());
-    assert(ctrl->IsInitialized());
-    SendControlMessage(move(ctrl));
+    // LOG(debug) << "OFI transport (" << fId << "):       OnSend: data=" << msg->GetData() << ",size=" << msg->GetSize();
 
-    if (size) {
-        // Receive and process control message
-        // auto ctrl2 = ReceiveControlMessage();
-        // assert(ctrl2->has_post_buffer_acknowledgement());
-        // assert(ctrl2->post_buffer_acknowledgement().size() == size);
+    boost::asio::mutable_buffer buffer(msg->GetData(), size);
 
-        // Send data
-        fi_context ctx;
-        auto ret = fi_send(fDataEndpoint, msg->GetData(), size, nullptr, fRemoteDataAddr, &ctx);
-        if (ret < 0)
-            throw SocketError(tools::ToString("Failed posting ofi send buffer, reason: ", fi_strerror(ret)));
+    if (fNeedOfiMemoryRegistration) {
+        asiofi::memory_region mr(*fOfiDomain, buffer, asiofi::mr::access::send);
+        auto desc = mr.desc();
+
+        fDataEndpoint->send(buffer, desc, [&, size, msg2 = move(msg), mr2 = move(mr)](boost::asio::mutable_buffer) mutable {
+            // LOG(debug) << "OFI transport (" << fId << "): >>>>> Data buffer sent";
+            fBytesTx += size;
+            fMessagesTx++;
+            fSendSem.async_signal([&](const boost::system::error_code& ec) {
+                if (!ec) {
+                    // LOG(debug) << "OFI transport (" << fId << "):     > Signal fSendSem=" << fSendSem.get_value();
+                }
+            });
+        });
+    } else {
+        fDataEndpoint->send(buffer, [&, size, msg2 = move(msg)](boost::asio::mutable_buffer) mutable {
+            // LOG(debug) << "OFI transport (" << fId << "): >>>>> Data buffer sent";
+            fBytesTx += size;
+            fMessagesTx++;
+            fSendSem.async_signal([&](const boost::system::error_code& ec) {
+                if (!ec) {
+                    // LOG(debug) << "OFI transport (" << fId << "):     > Signal fSendSem=" << fSendSem.get_value();
+                }
+            });
+        });
     }
 
-    if (size) {
-        fi_cq_err_entry cqEntry;
-        auto ret = fi_cq_sread(fDataCompletionQueueTx, &cqEntry, 1, nullptr, -1);
-        if (ret != 1)
-            throw SocketError(tools::ToString("Failed reading ofi tx completion queue event, reason: ", fi_strerror(ret)));
-    }
-
-    msg.reset(nullptr);
-    fBytesTx += size;
-    fMessagesTx++;
-
-    return size;
-}
-catch (const SilentSocketError& e)
-{
-    return -2;
-}
-catch (const std::exception& e)
-{
-    LOG(error) << e.what();
-    return -1;
+    // LOG(debug) << "OFI transport (" << fId << "): LEAVE OnSend";
 }
 
-auto Socket::ReceiveImpl(FairMQMessagePtr& msg, const int flags, const int timeout) -> int
+auto Socket::Receive(MessagePtr& msg, const int /*timeout*/) -> int
 try {
-    if (fWaitingForControlPeer) {
-        WaitForControlPeer(); 
-        AnnounceDataAddress();
-        ProcessDataAddressAnnouncement(ReceiveControlMessage());
-    }
+    // LOG(debug) << "OFI transport (" << fId << "): ENTER Receive";
+    azmq::message zmsg;
+    auto recv = fRecvQueueRead.receive(zmsg);
 
-    // Receive and process control message
-    auto ctrl = ReceiveControlMessage();
-    assert(ctrl->has_post_buffer());
-    auto postBuffer = ctrl->post_buffer();
-    auto size = postBuffer.size();
-
-    // Receive data
-    if (size) {
-        fi_context ctx;
-        msg->Rebuild(size);
-        auto buf = msg->GetData();
-        auto size2 = msg->GetSize();
-        auto ret = fi_recv(fDataEndpoint, buf, size2, nullptr, fRemoteDataAddr, &ctx);
-        if (ret < 0)
-            throw SocketError(tools::ToString("Failed posting ofi receive buffer, reason: ", fi_strerror(ret)));
-
-        // Create and send control message
-        // auto ctrl2 = tools::make_unique<ControlMessage>();
-        // auto ack = tools::make_unique<PostBufferAcknowledgement>();
-        // ack->set_size(msg->GetSize());
-        // ctrl2->set_allocated_post_buffer_acknowledgement(ack.release());
-        // assert(ctrl2->IsInitialized());
-        // SendControlMessage(move(ctrl2));
-
-        fi_cq_err_entry cqEntry;
-        ret = fi_cq_sread(fDataCompletionQueueRx, &cqEntry, 1, nullptr, -1);
-        if (ret != 1)
-            throw SocketError(tools::ToString("Failed reading ofi rx completion queue event, reason: ", fi_strerror(ret)));
-        assert(cqEntry.len == size2);
-        assert(cqEntry.buf == buf);
+    size_t size = 0;
+    if (recv > 0) {
+        msg = move(*(static_cast<MessagePtr*>(zmsg.buffer().data())));
+        size = msg->GetSize();
     }
 
     fBytesRx += size;
     fMessagesRx++;
 
+    // LOG(debug) << "OFI transport (" << fId << "): LEAVE Receive";
     return size;
-}
-catch (const SilentSocketError& e)
-{
-    return -2;
-}
-catch (const std::exception& e)
-{
+} catch (const exception& e) {
     LOG(error) << e.what();
     return -1;
+} catch (const boost::system::error_code& e) {
+    LOG(error) << e;
+    return -1;
 }
 
-auto Socket::SendImpl(vector<FairMQMessagePtr>& msgVec, const int flags, const int timeout) -> int64_t 
+auto Socket::Receive(vector<MessagePtr>& msgVec, const int timeout) -> int64_t
+{
+    return ReceiveImpl(msgVec, 0, timeout);
+}
+
+auto Socket::RecvQueueReader() -> void
+{
+    fRecvSem.async_wait([&](const boost::system::error_code& ec) {
+        if (!ec) {
+            static size_t size = fContext.GetSizeHint(); // temporary hack to provide expected message size for receive
+
+            auto msg = fContext.MakeReceiveMessage(size);
+            boost::asio::mutable_buffer buffer(msg->GetData(), size);
+
+            if (fNeedOfiMemoryRegistration) {
+                asiofi::memory_region mr(*fOfiDomain, buffer, asiofi::mr::access::recv);
+                auto desc = mr.desc();
+
+                fDataEndpoint->recv(buffer, desc, [&, msg2 = move(msg), mr2 = move(mr)](boost::asio::mutable_buffer) mutable {
+                    MessagePtr* msgptr(new std::unique_ptr<Message>(move(msg2)));
+                    fRecvQueueWrite.async_send(azmq::message(boost::asio::const_buffer(msgptr, sizeof(MessagePtr))), [&](const boost::system::error_code& ec2, size_t /*bytes_transferred2*/) {
+                        if (!ec2) {
+                            // LOG(debug) << "OFI transport (" << fId << "): <<<<< Data buffer received, bytes_transferred2=" << bytes_transferred2;
+                            fRecvSem.async_signal([&](const boost::system::error_code& ec3) {
+                                if (!ec3) {
+                                    // LOG(debug) << "OFI transport (" << fId << "):     < Signal fRecvSem";
+                                }
+                            });
+                        }
+                    });
+                });
+            } else {
+                fDataEndpoint->recv(buffer, [&, msg2 = move(msg)](boost::asio::mutable_buffer) mutable {
+                    MessagePtr* msgptr(new std::unique_ptr<Message>(move(msg2)));
+                    fRecvQueueWrite.async_send(azmq::message(boost::asio::const_buffer(msgptr, sizeof(MessagePtr))), [&](const boost::system::error_code& ec2, size_t /*bytes_transferred2*/) {
+                        if (!ec2) {
+                            // LOG(debug) << "OFI transport (" << fId << "): <<<<< Data buffer received, bytes_transferred2=" << bytes_transferred2;
+                            fRecvSem.async_signal([&](const boost::system::error_code& ec3) {
+                                if (!ec3) {
+                                    // LOG(debug) << "OFI transport (" << fId << "):     < Signal fRecvSem";
+                                }
+                            });
+                        }
+                    });
+                });
+            }
+
+            boost::asio::post(fContext.GetIoContext(), bind(&Socket::RecvQueueReader, this));
+        }
+    });
+}
+
+auto Socket::Send(vector<MessagePtr>& msgVec, const int timeout) -> int64_t
+{
+    return SendImpl(msgVec, 0, timeout);
+}
+
+auto Socket::SendImpl(vector<FairMQMessagePtr>& /*msgVec*/, const int /*flags*/, const int /*timeout*/) -> int64_t 
 {
     throw SocketError{"Not yet implemented."};
-    // const unsigned int vecSize = msgVec.size();
-    // int elapsed = 0;
-    //
-    // // Sending vector typicaly handles more then one part
-    // if (vecSize > 1)
-    // {
-    //     int64_t totalSize = 0;
-    //     int nbytes = -1;
-    //     bool repeat = false;
-    //
-    //     while (true && !fInterrupted)
-    //     {
-    //         for (unsigned int i = 0; i < vecSize; ++i)
-    //         {
-    //             nbytes = zmq_msg_send(static_cast<FairMQMessageSHM*>(msgVec[i].get())->GetMessage(),
-    //                                   fSocket,
-    //                                   (i < vecSize - 1) ? ZMQ_SNDMORE|flags : flags);
-    //             if (nbytes >= 0)
-    //             {
-    //                 static_cast<FairMQMessageSHM*>(msgVec[i].get())->fQueued = true;
-    //                 size_t size = msgVec[i]->GetSize();
-    //
-    //                 totalSize += size;
-    //             }
-    //             else
-    //             {
-    //                 // according to ZMQ docs, this can only occur for the first part
-    //                 if (zmq_errno() == EAGAIN)
-    //                 {
-    //                     if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
-    //                     {
-    //                         if (timeout)
-    //                         {
-    //                             elapsed += fSndTimeout;
-    //                             if (elapsed >= timeout)
-    //                             {
-    //                                 return -2;
-    //                             }
-    //                         }
-    //                         repeat = true;
-    //                         break;
-    //                     }
-    //                     else
-    //                     {
-    //                         return -2;
-    //                     }
-    //                 }
-    //                 if (zmq_errno() == ETERM)
-    //                 {
-    //                     LOG(info) << "terminating socket " << fId;
-    //                     return -1;
-    //                 }
-    //                 LOG(error) << "Failed sending on socket " << fId << ", reason: " << zmq_strerror(errno);
-    //                 return nbytes;
-    //             }
-    //         }
-    //
-    //         if (repeat)
-    //         {
-    //             continue;
-    //         }
-    //
-    //         // store statistics on how many messages have been sent (handle all parts as a single message)
-    //         ++fMessagesTx;
-    //         fBytesTx += totalSize;
-    //         return totalSize;
-    //     }
-    //
-    //     return -1;
-    // } // If there's only one part, send it as a regular message
-    // else if (vecSize == 1)
-    // {
-    //     return Send(msgVec.back(), flags);
-    // }
-    // else // if the vector is empty, something might be wrong
-    // {
-    //     LOG(warn) << "Will not send empty vector";
-    //     return -1;
-    // }
 }
 
-auto Socket::ReceiveImpl(vector<FairMQMessagePtr>& msgVec, const int flags, const int timeout) -> int64_t
+auto Socket::ReceiveImpl(vector<FairMQMessagePtr>& /*msgVec*/, const int /*flags*/, const int /*timeout*/) -> int64_t
 {
     throw SocketError{"Not yet implemented."};
-    // int64_t totalSize = 0;
-    // int64_t more = 0;
-    // bool repeat = false;
-    // int elapsed = 0;
-    //
-    // while (true)
-    // {
-    //     // Warn if the vector is filled before Receive() and empty it.
-    //     // if (msgVec.size() > 0)
-    //     // {
-    //     //     LOG(warn) << "Message vector contains elements before Receive(), they will be deleted!";
-    //     //     msgVec.clear();
-    //     // }
-    //
-    //     totalSize = 0;
-    //     more = 0;
-    //     repeat = false;
-    //
-    //     do
-    //     {
-    //         FairMQMessagePtr part(new FairMQMessageSHM(fManager, GetTransport()));
-    //         zmq_msg_t* msgPtr = static_cast<FairMQMessageSHM*>(part.get())->GetMessage();
-    //
-    //         int nbytes = zmq_msg_recv(msgPtr, fSocket, flags);
-    //         if (nbytes == 0)
-    //         {
-    //             msgVec.push_back(move(part));
-    //         }
-    //         else if (nbytes > 0)
-    //         {
-    //             MetaHeader* hdr = static_cast<MetaHeader*>(zmq_msg_data(msgPtr));
-    //             size_t size = 0;
-    //             static_cast<FairMQMessageSHM*>(part.get())->fHandle = hdr->fHandle;
-    //             static_cast<FairMQMessageSHM*>(part.get())->fSize = hdr->fSize;
-    //             static_cast<FairMQMessageSHM*>(part.get())->fRegionId = hdr->fRegionId;
-    //             static_cast<FairMQMessageSHM*>(part.get())->fHint = hdr->fHint;
-    //             size = part->GetSize();
-    //
-    //             msgVec.push_back(move(part));
-    //
-    //             totalSize += size;
-    //         }
-    //         else if (zmq_errno() == EAGAIN)
-    //         {
-    //             if (!fInterrupted && ((flags & ZMQ_DONTWAIT) == 0))
-    //             {
-    //                 if (timeout)
-    //                 {
-    //                     elapsed += fSndTimeout;
-    //                     if (elapsed >= timeout)
-    //                     {
-    //                         return -2;
-    //                     }
-    //                 }
-    //                 repeat = true;
-    //                 break;
-    //             }
-    //             else
-    //             {
-    //                 return -2;
-    //             }
-    //         }
-    //         else
-    //         {
-    //             return nbytes;
-    //         }
-    //
-    //         size_t more_size = sizeof(more);
-    //         zmq_getsockopt(fSocket, ZMQ_RCVMORE, &more, &more_size);
-    //     }
-    //     while (more);
-    //
-    //     if (repeat)
-    //     {
-    //         continue;
-    //     }
-    //
-    //     // store statistics on how many messages have been received (handle all parts as a single message)
-    //     ++fMessagesRx;
-    //     fBytesRx += totalSize;
-    //     return totalSize;
+}
+
+auto Socket::Close() -> void {}
+
+auto Socket::SetOption(const string& /*option*/, const void* /*value*/, size_t /*valueSize*/) -> void
+{
+    // if (zmq_setsockopt(fControlSocket, GetConstant(option), value, valueSize) < 0) {
+        // throw SocketError{tools::ToString("Failed setting socket option, reason: ", zmq_strerror(errno))};
     // }
 }
 
-auto Socket::Close() -> void
+auto Socket::GetOption(const string& /*option*/, void* /*value*/, size_t* /*valueSize*/) -> void
 {
-    if (zmq_close(fControlSocket) != 0)
-        throw SocketError(tools::ToString("Failed closing zmq meta socket, reason: ", zmq_strerror(errno)));
-
-    if (zmq_close(fMonitorSocket) != 0)
-        throw SocketError(tools::ToString("Failed closing zmq monitor socket, reason: ", zmq_strerror(errno)));
-
-	if (fDataEndpoint) {
-        auto ret = fi_close(&fDataEndpoint->fid);
-        if (ret != FI_SUCCESS)
-            throw SocketError(tools::ToString("Failed closing ofi endpoint, reason: ", fi_strerror(ret)));
-    }
-
-	if (fDataCompletionQueueTx) {
-        auto ret = fi_close(&fDataCompletionQueueTx->fid);
-        if (ret != FI_SUCCESS)
-            throw SocketError(tools::ToString("Failed closing ofi transmit completion queue, reason: ", fi_strerror(ret)));
-    }
-
-	if (fDataCompletionQueueRx) {
-        auto ret = fi_close(&fDataCompletionQueueRx->fid);
-        if (ret != FI_SUCCESS)
-            throw SocketError(tools::ToString("Failed closing ofi receive completion queue, reason: ", fi_strerror(ret)));
-    }
+    // if (zmq_getsockopt(fControlSocket, GetConstant(option), value, valueSize) < 0) {
+        // throw SocketError{tools::ToString("Failed getting socket option, reason: ", zmq_strerror(errno))};
+    // }
 }
 
-auto Socket::SetOption(const string& option, const void* value, size_t valueSize) -> void
+void Socket::SetLinger(const int /*value*/)
 {
-    if (zmq_setsockopt(fControlSocket, GetConstant(option), value, valueSize) < 0) {
-        throw SocketError{tools::ToString("Failed setting socket option, reason: ", zmq_strerror(errno))};
-    }
-}
-
-auto Socket::GetOption(const string& option, void* value, size_t* valueSize) -> void
-{
-    if (zmq_getsockopt(fControlSocket, GetConstant(option), value, valueSize) < 0) {
-        throw SocketError{tools::ToString("Failed getting socket option, reason: ", zmq_strerror(errno))};
-    }
+    // azmq::socket::linger opt(value);
+    // fControlEndpoint.set_option(opt);
 }
 
 int Socket::GetLinger() const
 {
-    int value = 0;
-    size_t valueSize;
-    if (zmq_getsockopt(fControlSocket, ZMQ_LINGER, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_LINGER, reason: ", zmq_strerror(errno)));
-    }
-    return value;
+    // azmq::socket::linger opt(0);
+    // fControlEndpoint.get_option(opt);
+    // return opt.value();
+    return 0;
 }
 
-void Socket::SetSndBufSize(const int value)
+void Socket::SetSndBufSize(const int /*value*/)
 {
-    if (zmq_setsockopt(fControlSocket, ZMQ_SNDHWM, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed setting ZMQ_SNDHWM, reason: ", zmq_strerror(errno)));
-    }
+    // azmq::socket::snd_hwm opt(value);
+    // fControlEndpoint.set_option(opt);
 }
 
 int Socket::GetSndBufSize() const
 {
-    int value = 0;
-    size_t valueSize;
-    if (zmq_getsockopt(fControlSocket, ZMQ_SNDHWM, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_SNDHWM, reason: ", zmq_strerror(errno)));
-    }
-    return value;
+    // azmq::socket::snd_hwm opt(0);
+    // fControlEndpoint.get_option(opt);
+    // return opt.value();
+    return 0;
 }
 
-void Socket::SetRcvBufSize(const int value)
+void Socket::SetRcvBufSize(const int /*value*/)
 {
-    if (zmq_setsockopt(fControlSocket, ZMQ_RCVHWM, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed setting ZMQ_RCVHWM, reason: ", zmq_strerror(errno)));
-    }
+    // azmq::socket::rcv_hwm opt(value);
+    // fControlEndpoint.set_option(opt);
 }
 
 int Socket::GetRcvBufSize() const
 {
-    int value = 0;
-    size_t valueSize;
-    if (zmq_getsockopt(fControlSocket, ZMQ_RCVHWM, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_RCVHWM, reason: ", zmq_strerror(errno)));
-    }
-    return value;
+    // azmq::socket::rcv_hwm opt(0);
+    // fControlEndpoint.get_option(opt);
+    // return opt.value();
+    return 0;
 }
 
-void Socket::SetSndKernelSize(const int value)
+void Socket::SetSndKernelSize(const int /*value*/)
 {
-    if (zmq_setsockopt(fControlSocket, ZMQ_SNDBUF, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_SNDBUF, reason: ", zmq_strerror(errno)));
-    }
+    // azmq::socket::snd_buf opt(value);
+    // fControlEndpoint.set_option(opt);
 }
 
 int Socket::GetSndKernelSize() const
 {
-    int value = 0;
-    size_t valueSize;
-    if (zmq_getsockopt(fControlSocket, ZMQ_SNDBUF, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_SNDBUF, reason: ", zmq_strerror(errno)));
-    }
-    return value;
+    // azmq::socket::snd_buf opt(0);
+    // fControlEndpoint.get_option(opt);
+    // return opt.value();
+    return 0;
 }
 
-void Socket::SetRcvKernelSize(const int value)
+void Socket::SetRcvKernelSize(const int /*value*/)
 {
-    if (zmq_setsockopt(fControlSocket, ZMQ_RCVBUF, &value, sizeof(value)) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_RCVBUF, reason: ", zmq_strerror(errno)));
-    }
+    // azmq::socket::rcv_buf opt(value);
+    // fControlEndpoint.set_option(opt);
 }
 
 int Socket::GetRcvKernelSize() const
 {
-    int value = 0;
-    size_t valueSize;
-    if (zmq_getsockopt(fControlSocket, ZMQ_RCVBUF, &value, &valueSize) < 0) {
-        throw SocketError(tools::ToString("failed getting ZMQ_RCVBUF, reason: ", zmq_strerror(errno)));
-    }
-    return value;
+    // azmq::socket::rcv_buf opt(0);
+    // fControlEndpoint.get_option(opt);
+    // return opt.value();
+    return 0;
 }
 
 auto Socket::GetConstant(const string& constant) -> int

fairmq/ofi/Socket.h

@@ -12,13 +12,16 @@
 #include <FairMQSocket.h>
 #include <FairMQMessage.h>
 #include <fairmq/ofi/Context.h>
-#include <fairmq/ofi/Control.pb.h>
+#include <fairmq/ofi/ControlMessages.h>
 
+#include <asiofi/connected_endpoint.hpp>
+#include <asiofi/memory_resources.hpp>
+#include <asiofi/passive_endpoint.hpp>
+#include <asiofi/semaphore.hpp>
+#include <azmq/socket.hpp>
 #include <boost/asio.hpp>
 #include <memory> // unique_ptr
 #include <netinet/in.h>
-#include <rdma/fabric.h>
-class FairMQTransportFactory;
 
 namespace fair
 {
@@ -36,7 +39,7 @@ namespace ofi
 class Socket final : public fair::mq::Socket
 {
   public:
-    Socket(Context& factory, const std::string& type, const std::string& name, const std::string& id = "", FairMQTransportFactory* fac);
+    Socket(Context& context, const std::string& type, const std::string& name, const std::string& id = "");
     Socket(const Socket&) = delete;
     Socket operator=(const Socket&) = delete;
 
@@ -50,12 +53,7 @@ class Socket final : public fair::mq::Socket
     auto Send(std::vector<MessagePtr>& msgVec, int timeout = 0) -> int64_t override;
     auto Receive(std::vector<MessagePtr>& msgVec, int timeout = 0) -> int64_t override;
 
-    auto TrySend(MessagePtr& msg) -> int override;
-    auto TryReceive(MessagePtr& msg) -> int override;
-    auto TrySend(std::vector<MessagePtr>& msgVec) -> int64_t override;
-    auto TryReceive(std::vector<MessagePtr>& msgVec) -> int64_t override;
-
-    auto GetSocket() const -> void* { return fControlSocket; }
+    auto GetSocket() const -> void* { return nullptr; }
 
     void SetLinger(const int value) override;
     int GetLinger() const override;
@@ -83,43 +81,39 @@ class Socket final : public fair::mq::Socket
     ~Socket() override;
 
   private:
-    void* fControlSocket;
-    void* fMonitorSocket;
-    fid_ep* fDataEndpoint;
-    fid_cq* fDataCompletionQueueTx;
-    fid_cq* fDataCompletionQueueRx;
+    Context& fContext;
+    std::unique_ptr<asiofi::info> fOfiInfo;
+    std::unique_ptr<asiofi::fabric> fOfiFabric;
+    std::unique_ptr<asiofi::domain> fOfiDomain;
+    std::unique_ptr<asiofi::passive_endpoint> fPassiveEndpoint;
+    std::unique_ptr<asiofi::connected_endpoint> fDataEndpoint;
     std::string fId;
     std::atomic<unsigned long> fBytesTx;
     std::atomic<unsigned long> fBytesRx;
     std::atomic<unsigned long> fMessagesTx;
     std::atomic<unsigned long> fMessagesRx;
-    Context& fContext;
-    fi_addr_t fRemoteDataAddr;
-    sockaddr_in fLocalDataAddr;
-    bool fWaitingForControlPeer;
-    boost::asio::io_service::strand fIoStrand;
-
+    Address fRemoteAddr;
+    Address fLocalAddr;
     int fSndTimeout;
     int fRcvTimeout;
+    azmq::socket fRecvQueueWrite, fRecvQueueRead;
+    asiofi::semaphore fSendSem, fRecvSem;
+    std::atomic<bool> fNeedOfiMemoryRegistration;
+    std::atomic<bool> fBound;
+    std::atomic<bool> fConnected;
 
-    auto SendImpl(MessagePtr& msg, const int flags, const int timeout) -> int;
+    auto OnSend(MessagePtr& msg) -> void;
+    auto RecvQueueReader() -> void;
     auto ReceiveImpl(MessagePtr& msg, const int flags, const int timeout) -> int;
     auto SendImpl(std::vector<MessagePtr>& msgVec, const int flags, const int timeout) -> int64_t;
     auto ReceiveImpl(std::vector<MessagePtr>& msgVec, const int flags, const int timeout) -> int64_t;
 
-    auto InitDataEndpoint() -> void;
-    auto WaitForControlPeer() -> void;
-    auto AnnounceDataAddress() -> void;
-    auto SendControlMessage(std::unique_ptr<ControlMessage> ctrl) -> void;
-    auto ReceiveControlMessage() -> std::unique_ptr<ControlMessage>;
-    auto ProcessDataAddressAnnouncement(std::unique_ptr<ControlMessage> ctrl) -> void;
-    auto ConnectControlSocket(Context::Address address) -> void;
-    auto BindControlSocket(Context::Address address) -> void;
+    // auto WaitForControlPeer() -> void;
+    // auto AnnounceDataAddress() -> void;
+    auto InitOfi(Address addr) -> void;
+    // auto ReceiveDataAddressAnnouncement() -> void;
 }; /* class Socket */
 
-// helper function to clean up the object holding the data after it is transported.
-void free_string(void* /*data*/, void* hint);
-
 struct SilentSocketError : SocketError { using SocketError::SocketError; };
 
 } /* namespace ofi */

fairmq/ofi/TransportFactory.cxx

@@ -24,59 +24,68 @@ namespace ofi
 using namespace std;
 
 TransportFactory::TransportFactory(const string& id, const FairMQProgOptions* config)
-try : FairMQTransportFactory{id}
-{
-    LOG(debug) << "Transport: Using ZeroMQ (" << fContext.GetZmqVersion() << ") & "
-               << "OFI libfabric (API " << fContext.GetOfiApiVersion() << ") & "
-               << "Google Protobuf (" << fContext.GetPbVersion() << ") & "
-               << "Boost.Asio (" << fContext.GetBoostVersion() << ")";
-}
-catch (ContextError& e)
+try : FairMQTransportFactory(id)
+    , fContext(*this, *this, 1)
 {
+    LOG(debug) << "OFI transport: Using AZMQ & "
+               << "asiofi (" << fContext.GetAsiofiVersion() << ")";
+    if (config) {
+        fContext.SetSizeHint(config->GetValue<size_t>("ofi-size-hint")); // temporary hack to provide expected message size for receive
+    }
+} catch (ContextError& e) {
     throw TransportFactoryError{e.what()};
 }
 
-auto TransportFactory::CreateMessage() const -> MessagePtr
+auto TransportFactory::CreateMessage() -> MessagePtr
 {
-    return MessagePtr{new Message()};
+    return MessagePtr{new Message(&fMemoryResource)};
 }
 
-auto TransportFactory::CreateMessage(const size_t size) const -> MessagePtr
+auto TransportFactory::CreateMessage(const size_t size) -> MessagePtr
 {
-    return MessagePtr{new Message(size)};
+    return MessagePtr{new Message(&fMemoryResource, size)};
 }
 
-auto TransportFactory::CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint) const -> MessagePtr
+auto TransportFactory::CreateMessage(void* data,
+                                     const size_t size,
+                                     fairmq_free_fn* ffn,
+                                     void* hint) -> MessagePtr
 {
-    return MessagePtr{new Message(data, size, ffn, hint)};
+    return MessagePtr{new Message(&fMemoryResource, data, size, ffn, hint)};
 }
 
-auto TransportFactory::CreateMessage(UnmanagedRegionPtr& region, void* data, const size_t size, void* hint) const -> MessagePtr
+auto TransportFactory::CreateMessage(UnmanagedRegionPtr& region,
+                                     void* data,
+                                     const size_t size,
+                                     void* hint) -> MessagePtr
 {
-    return MessagePtr{new Message(region, data, size, hint)};
+    return MessagePtr{new Message(&fMemoryResource, region, data, size, hint)};
 }
 
 auto TransportFactory::CreateSocket(const string& type, const string& name) -> SocketPtr
 {
-    return SocketPtr{new Socket(fContext, type, name, GetId(), this)};
+    return SocketPtr{new Socket(fContext, type, name, GetId())};
 }
 
-auto TransportFactory::CreatePoller(const vector<FairMQChannel>& channels) const -> PollerPtr
+auto TransportFactory::CreatePoller(const vector<FairMQChannel>& /*channels*/) const -> PollerPtr
 {
-    return PollerPtr{new Poller(channels)};
+    throw runtime_error{"Not yet implemented (Poller)."};
+    // return PollerPtr{new Poller(channels)};
 }
 
-auto TransportFactory::CreatePoller(const vector<const FairMQChannel*>& channels) const -> PollerPtr
+auto TransportFactory::CreatePoller(const vector<FairMQChannel*>& /*channels*/) const -> PollerPtr
 {
-    return PollerPtr{new Poller(channels)};
+    throw runtime_error{"Not yet implemented (Poller)."};
+    // return PollerPtr{new Poller(channels)};
 }
 
-auto TransportFactory::CreatePoller(const unordered_map<string, vector<FairMQChannel>>& channelsMap, const vector<string>& channelList) const -> PollerPtr
+auto TransportFactory::CreatePoller(const unordered_map<string, vector<FairMQChannel>>& /*channelsMap*/, const vector<string>& /*channelList*/) const -> PollerPtr
 {
-    return PollerPtr{new Poller(channelsMap, channelList)};
+    throw runtime_error{"Not yet implemented (Poller)."};
+    // return PollerPtr{new Poller(channelsMap, channelList)};
 }
 
-auto TransportFactory::CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback) const -> UnmanagedRegionPtr
+auto TransportFactory::CreateUnmanagedRegion(const size_t /*size*/, FairMQRegionCallback /*callback*/) const -> UnmanagedRegionPtr
 {
     throw runtime_error{"Not yet implemented UMR."};
 }

fairmq/ofi/TransportFactory.h

@@ -13,6 +13,8 @@
 #include <options/FairMQProgOptions.h>
 #include <fairmq/ofi/Context.h>
 
+#include <asiofi.hpp>
+
 namespace fair
 {
 namespace mq
@@ -22,7 +24,7 @@ namespace ofi
 
 /**
  * @class TransportFactory TransportFactory.h <fairmq/ofi/TransportFactory.h>
- * @brief FairMQ transport factory for the ofi transport (implemented with ZeroMQ + libfabric)
+ * @brief FairMQ transport factory for the ofi transport
  *
  * @todo TODO insert long description
  */
@@ -33,27 +35,28 @@ class TransportFactory final : public FairMQTransportFactory
     TransportFactory(const TransportFactory&) = delete;
     TransportFactory operator=(const TransportFactory&) = delete;
 
-    auto CreateMessage() const -> MessagePtr override;
-    auto CreateMessage(const std::size_t size) const -> MessagePtr override;
-    auto CreateMessage(void* data, const std::size_t size, fairmq_free_fn* ffn, void* hint = nullptr) const -> MessagePtr override;
-    auto CreateMessage(UnmanagedRegionPtr& region, void* data, const std::size_t size, void* hint = nullptr) const -> MessagePtr override;
+    auto CreateMessage() -> MessagePtr override;
+    auto CreateMessage(const std::size_t size) -> MessagePtr override;
+    auto CreateMessage(void* data, const std::size_t size, fairmq_free_fn* ffn, void* hint = nullptr) -> MessagePtr override;
+    auto CreateMessage(UnmanagedRegionPtr& region, void* data, const std::size_t size, void* hint = nullptr) -> MessagePtr override;
 
     auto CreateSocket(const std::string& type, const std::string& name) -> SocketPtr override;
 
     auto CreatePoller(const std::vector<FairMQChannel>& channels) const -> PollerPtr override;
-    auto CreatePoller(const std::vector<const FairMQChannel*>& channels) const -> PollerPtr override;
+    auto CreatePoller(const std::vector<FairMQChannel*>& channels) const -> PollerPtr override;
     auto CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const -> PollerPtr override;
 
     auto CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback = nullptr) const -> UnmanagedRegionPtr override;
 
     auto GetType() const -> Transport override;
 
-    void Interrupt() override {}
-    void Resume() override {}
-    void Reset() override {}
+    void Interrupt() override { fContext.Interrupt(); }
+    void Resume() override { fContext.Resume(); }
+    void Reset() override { fContext.Reset(); }
 
   private:
     mutable Context fContext;
+    asiofi::allocated_pool_resource fMemoryResource;
 }; /* class TransportFactory */  
 
 } /* namespace ofi */

fairmq/options/FairMQProgOptions.cxx

@@ -56,17 +56,19 @@ FairMQProgOptions::FairMQProgOptions()
         ("print-options", po::value<bool  >()->implicit_value(true),    "Print options in machine-readable format (<option>:<computed-value>:<type>:<description>)");
 
     fMQOptions.add_options()
-        ("id",                     po::value<string>(),                            "Device ID (required argument).")
-        ("io-threads",             po::value<int   >()->default_value(1),          "Number of I/O threads.")
-        ("transport",              po::value<string>()->default_value("zeromq"),   "Transport ('zeromq'/'nanomsg'/'shmem')  .")
-        ("network-interface",      po::value<string>()->default_value("default"),  "Network interface to bind on (e.g. eth0, ib0..., default will try to detect the interface of the default route).")
-        ("config-key",             po::value<string>(),                            "Use provided value instead of device id for fetching the configuration from the config file.")
-        ("initialization-timeout", po::value<int   >()->default_value(120),        "Timeout for the initialization in seconds (when expecting dynamic initialization).")
-        ("print-channels",         po::value<bool  >()->implicit_value(true),      "Print registered channel endpoints in a machine-readable format (<channel name>:<min num subchannels>:<max num subchannels>)")
-        ("shm-segment-size",       po::value<size_t>()->default_value(2000000000), "Shared memory: size of the shared memory segment (in bytes).")
-        ("shm-monitor",            po::value<bool  >()->default_value(true),       "Shared memory: run monitor daemon.")
-        ("rate",                   po::value<float >()->default_value(0.),         "Rate for conditional run loop (Hz).")
-        ("session",                po::value<string>()->default_value("default"),  "Session name.");
+        ("id",                     po::value<string  >(),                            "Device ID (required argument).")
+        ("io-threads",             po::value<int     >()->default_value(1),          "Number of I/O threads.")
+        ("transport",              po::value<string  >()->default_value("zeromq"),   "Transport ('zeromq'/'nanomsg'/'shmem').")
+        ("network-interface",      po::value<string  >()->default_value("default"),  "Network interface to bind on (e.g. eth0, ib0..., default will try to detect the interface of the default route).")
+        ("config-key",             po::value<string  >(),                            "Use provided value instead of device id for fetching the configuration from the config file.")
+        ("initialization-timeout", po::value<int     >()->default_value(120),        "Timeout for the initialization in seconds (when expecting dynamic initialization).")
+        ("max-run-time",           po::value<uint64_t>()->default_value(0),          "Maximum runtime for the Running state handler, after which state will change to Ready (in seconds, 0 for no limit).")
+        ("print-channels",         po::value<bool    >()->implicit_value(true),      "Print registered channel endpoints in a machine-readable format (<channel name>:<min num subchannels>:<max num subchannels>)")
+        ("shm-segment-size",       po::value<size_t  >()->default_value(2000000000), "Shared memory: size of the shared memory segment (in bytes).")
+        ("shm-monitor",            po::value<bool    >()->default_value(true),       "Shared memory: run monitor daemon.")
+        ("ofi-size-hint",          po::value<size_t  >()->default_value(2000000),    "EXPERIMENTAL: OFI size hint for the allocator.")
+        ("rate",                   po::value<float   >()->default_value(0.),         "Rate for conditional run loop (Hz).")
+        ("session",                po::value<string  >()->default_value("default"),  "Session name.");
 
     fParserOptions.add_options()
         ("mq-config",      po::value<string>(),                                    "JSON input as file.")
@@ -452,7 +454,7 @@ int FairMQProgOptions::PrintOptions()
            << "\n";
     }
 
-    LOG(info) << ss.str();
+    LOG(debug) << ss.str();
 
     return 0;
 }

fairmq/plugins/Control.cxx

@@ -14,6 +14,8 @@
 #include <cstdlib>
 #include <functional>
 #include <atomic>
+#include <thread>
+#include <chrono>
 
 using namespace std;
 
@@ -53,8 +55,8 @@ Control::Control(const string& name, const Plugin::Version version, const string
     , fDeviceHasShutdown(false)
     , fPluginShutdownRequested(false)
 {
-    SubscribeToDeviceStateChange([&](DeviceState newState)
-    {
+    SubscribeToDeviceStateChange([&](DeviceState newState) {
+        LOG(trace) << "control plugin notified on new state: " << newState;
         {
             lock_guard<mutex> lock{fEventsMutex};
             fEvents.push(newState);
@@ -62,47 +64,52 @@ Control::Control(const string& name, const Plugin::Version version, const string
         fNewEvent.notify_one();
     });
 
-    try
-    {
+    try {
         TakeDeviceControl();
 
         auto control = GetProperty<string>("control");
 
-        if (control == "static")
-        {
+        if (control == "static") {
             LOG(debug) << "Running builtin controller: static";
             fControllerThread = thread(&Control::StaticMode, this);
-        }
-        else if (control == "interactive")
-        {
+        } else if (control == "interactive") {
             LOG(debug) << "Running builtin controller: interactive";
             fControllerThread = thread(&Control::InteractiveMode, this);
-        }
-        else
-        {
+        } else {
             LOG(error) << "Unrecognized control mode '" << control << "' requested. " << "Ignoring and falling back to static control mode.";
             fControllerThread = thread(&Control::StaticMode, this);
         }
-    }
-    catch (PluginServices::DeviceControlError& e)
-    {
+    } catch (PluginServices::DeviceControlError& e) {
         // If we are here, it means another plugin has taken control. That's fine, just print the exception message and do nothing else.
         LOG(debug) << e.what();
     }
 
-    LOG(debug) << "catch-signals: " << GetProperty<int>("catch-signals");
-    if (GetProperty<int>("catch-signals") > 0)
-    {
+    if (GetProperty<int>("catch-signals") > 0) {
+        LOG(debug) << "Plugin '" << name << "' is setting up signal handling for SIGINT and SIGTERM";
         fSignalHandlerThread = thread(&Control::SignalHandler, this);
         signal(SIGINT, signal_handler);
         signal(SIGTERM, signal_handler);
-    }
-    else
-    {
+    } else {
         LOG(warn) << "Signal handling (e.g. Ctrl-C) has been deactivated.";
     }
 }
 
+auto Control::RunStartupSequence() -> void
+{
+    ChangeDeviceState(DeviceStateTransition::InitDevice);
+    while (WaitForNextState() != DeviceState::InitializingDevice) {}
+    ChangeDeviceState(DeviceStateTransition::CompleteInit);
+    while (WaitForNextState() != DeviceState::Initialized) {}
+    ChangeDeviceState(DeviceStateTransition::Bind);
+    while (WaitForNextState() != DeviceState::Bound) {}
+    ChangeDeviceState(DeviceStateTransition::Connect);
+    while (WaitForNextState() != DeviceState::DeviceReady) {}
+    ChangeDeviceState(DeviceStateTransition::InitTask);
+    while (WaitForNextState() != DeviceState::Ready) {}
+    ChangeDeviceState(DeviceStateTransition::Run);
+    while (WaitForNextState() != DeviceState::Running) {}
+}
+
 auto ControlPluginProgramOptions() -> Plugin::ProgOptions
 {
     namespace po = boost::program_options;
@@ -135,8 +142,7 @@ struct terminal_config
 };
 
 auto Control::InteractiveMode() -> void
-try
-{
+try {
     RunStartupSequence();
 
     char input; // hold the user console input
@@ -147,129 +153,226 @@ try
 
     terminal_config tconfig;
 
-    PrintInteractiveHelp();
+    bool color = GetProperty<bool>("color");
+
+    if (color) {
+        PrintInteractiveHelpColor();
+    } else {
+        PrintInteractiveHelp();
+    }
 
     bool keepRunning = true;
 
-    while (keepRunning)
-    {
-        if (poll(cinfd, 1, 500))
-        {
-            if (fDeviceShutdownRequested)
-            {
+    while (keepRunning) {
+        if (poll(cinfd, 1, 500)) {
+            if (fDeviceShutdownRequested) {
                 break;
             }
 
             cin >> input;
 
-            switch (input)
-            {
+            switch (input) {
+                case 'c':
+                    cout << "\n --> [i] check current state\n\n" << flush;
+                    LOG(state) << GetCurrentDeviceState();
+                break;
                 case 'i':
-                    LOG(info) << "\n\n --> [i] init device\n";
+                    cout << "\n --> [i] init device\n\n" << flush;
                     ChangeDeviceState(DeviceStateTransition::InitDevice);
+                    while (WaitForNextState() != DeviceState::InitializingDevice) {}
+                    ChangeDeviceState(DeviceStateTransition::CompleteInit);
+                break;
+                case 'b':
+                    cout << "\n --> [b] bind\n\n" << flush;
+                    ChangeDeviceState(DeviceStateTransition::Bind);
+                break;
+                case 'x':
+                    cout << "\n --> [x] connect\n\n" << flush;
+                    ChangeDeviceState(DeviceStateTransition::Connect);
                 break;
                 case 'j':
-                    LOG(info) << "\n\n --> [j] init task\n";
+                    cout << "\n --> [j] init task\n\n" << flush;
                     ChangeDeviceState(DeviceStateTransition::InitTask);
                 break;
-                case 'p':
-                    LOG(info) << "\n\n --> [p] pause\n";
-                    ChangeDeviceState(DeviceStateTransition::Pause);
-                break;
                 case 'r':
-                    LOG(info) << "\n\n --> [r] run\n";
+                    cout << "\n --> [r] run\n\n" << flush;
                     ChangeDeviceState(DeviceStateTransition::Run);
                 break;
                 case 's':
-                    LOG(info) << "\n\n --> [s] stop\n";
+                    cout << "\n --> [s] stop\n\n" << flush;
                     ChangeDeviceState(DeviceStateTransition::Stop);
                 break;
                 case 't':
-                    LOG(info) << "\n\n --> [t] reset task\n";
+                    cout << "\n --> [t] reset task\n\n" << flush;
                     ChangeDeviceState(DeviceStateTransition::ResetTask);
                 break;
                 case 'd':
-                    LOG(info) << "\n\n --> [d] reset device\n";
+                    cout << "\n --> [d] reset device\n\n" << flush;
                     ChangeDeviceState(DeviceStateTransition::ResetDevice);
                 break;
                 case 'k':
-                    LOG(info) << "\n\n --> [k] increase log severity\n";
+                    cout << "\n --> [k] increase log severity\n\n" << flush;
                     CycleLogConsoleSeverityUp();
                 break;
                 case 'l':
-                    LOG(info) << "\n\n --> [l] decrease log severity\n";
+                    cout << "\n --> [l] decrease log severity\n\n" << flush;
                     CycleLogConsoleSeverityDown();
                 break;
                 case 'n':
-                    LOG(info) << "\n\n --> [n] increase log verbosity\n";
+                    cout << "\n --> [n] increase log verbosity\n\n" << flush;
                     CycleLogVerbosityUp();
                 break;
                 case 'm':
-                    LOG(info) << "\n\n --> [m] decrease log verbosity\n";
+                    cout << "\n --> [m] decrease log verbosity\n\n" << flush;
                     CycleLogVerbosityDown();
                 break;
                 case 'h':
-                    LOG(info) << "\n\n --> [h] help\n";
-                    PrintInteractiveHelp();
+                    cout << "\n --> [h] help\n\n" << flush;
+                    if (color) {
+                        PrintInteractiveHelpColor();
+                        PrintStateMachineColor();
+                    } else {
+                        PrintInteractiveHelp();
+                        PrintStateMachine();
+                    }
                 break;
-                // case 'x':
-                //     LOG(info) << "\n\n --> [x] ERROR\n";
-                //     ChangeDeviceState(DeviceStateTransition::ERROR_FOUND);
-                //     break;
                 case 'q':
-                    LOG(info) << "\n\n --> [q] end\n";
+                    cout << "\n --> [q] end\n\n" << flush;
                     keepRunning = false;
                 break;
                 default:
-                    LOG(info) << "Invalid input: [" << input << "]";
+                    cout << "Invalid input: [" << input << "]. " << flush;
                     PrintInteractiveHelp();
                 break;
             }
         }
 
-        if (GetCurrentDeviceState() == DeviceState::Error)
-        {
+        if (GetCurrentDeviceState() == DeviceState::Error) {
             throw DeviceErrorState("Controlled device transitioned to error state.");
         }
 
-        if (fDeviceShutdownRequested)
-        {
+        if (fDeviceShutdownRequested) {
             break;
         }
     }
 
     RunShutdownSequence();
-}
-catch (PluginServices::DeviceControlError& e)
-{
+} catch (PluginServices::DeviceControlError& e) {
     // If we are here, it means another plugin has taken control. That's fine, just print the exception message and do nothing else.
     LOG(debug) << e.what();
+} catch (DeviceErrorState&) {
 }
-catch (DeviceErrorState&)
+
+auto Control::PrintInteractiveHelpColor() -> void
 {
+    stringstream ss;
+    ss << "Following control commands are available:\n\n"
+       << " [\033[01;32mh\033[0m] help, [\033[01;32mc\033[0m] check current device state,\n"
+       << " [\033[01;32mi\033[0m] init device, [\033[01;32mb\033[0m] bind, [\033[01;32mx\033[0m] connect, [\033[01;32mj\033[0m] init task,"
+       << " [\033[01;32mr\033[0m] run, [\033[01;32ms\033[0m] stop,\n"
+       << " [\033[01;32mt\033[0m] reset task, [\033[01;32md\033[0m] reset device, [\033[01;32mq\033[0m] end,\n"
+       << " [\033[01;32mk\033[0m] increase log severity [\033[01;32ml\033[0m] decrease log severity [\033[01;32mn\033[0m] increase log verbosity [\033[01;32mm\033[0m] decrease log verbosity\n\n";
+    cout << ss.str() << flush;
 }
 
 auto Control::PrintInteractiveHelp() -> void
 {
     stringstream ss;
-    ss << "\nFollowing control commands are available:\n\n"
-       << "[h] help, [p] pause, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device\n"
-       << "[k] increase log severity [l] decrease log severity [n] increase log verbosity [m] decrease log verbosity\n\n";
+    ss << "Following control commands are available:\n\n"
+       << " [h] help, [c] check current device state,\n"
+       << " [i] init device, [b] bind, [x] connect, [j] init task,\n"
+       << " [r] run, [s] stop,\n"
+       << " [t] reset task, [d] reset device, [q] end,\n"
+       << " [k] increase log severity, [l] decrease log severity, [n] increase log verbosity, [m] decrease log verbosity.\n\n";
+    cout << ss.str() << flush;
+}
+
+void Control::PrintStateMachineColor()
+{
+    stringstream ss;
+    ss << "               @                                \n"
+       << "   ┌───────────╨─────────────┐  ┌───────────┐   \n"
+       << "   │           \033[01;36mIDLE\033[0m         [\033[01;32mq\033[0m]─▶  \033[01;33mEXITING\033[0m  │   \n"
+       << "   └[\033[01;32mi\033[0m]─────────────────── ▲ ┘  └───────────┘   \n"
+       << " ╔══ ▼ ════════════════╗ ╔═╩══════════════════╗ \n"
+       << " ║ \033[01;33mINITIALIZING DEVICE\033[0m ║ ║  \033[01;33mRESETTING DEVICE\033[0m  ║ \n"
+       << " ╚══════════╦══════════╝ ╚════════ ▲ ═════════╝ \n"
+       << " ┌───────── ▼ ─────────┐           │  ┌────────────────────────────┐ \n"
+       << " │     \033[01;36mINITIALIZED\033[0m     │           │  │ Legend:                    │ \n"
+       << " └─────────[\033[01;32mb\033[0m]─────────┘           │  │----------------------------│ \n"
+       << " ╔═════════ ▼ ═════════╗           │  │ [\033[01;32mk\033[0m] keyboard shortcut for  │ \n"
+       << " ║       \033[01;33mBINDING\033[0m       ║           │  │     interactive controller │ \n"
+       << " ╚══════════╦══════════╝           │  │ ┌────────────────────────┐ │ \n"
+       << " ┌───────── ▼ ─────────┐           │  │ │      \033[01;36mIDLING STATE\033[0m      │ │ \n"
+       << " │        \033[01;36mBOUND\033[0m        │           │  │ └────────────────────────┘ │ \n"
+       << " └─────────[\033[01;32mx\033[0m]─────────┘           │  │ ╔════════════════════════╗ │ \n"
+       << " ╔═════════ ▼ ═════════╗           │  │ ║      \033[01;33mWORKING STATE\033[0m     ║ │ \n"
+       << " ║     \033[01;33mCONNECTING\033[0m      ║           │  │ ╚════════════════════════╝ │ \n"
+       << " ╚══════════╦══════════╝           │  └────────────────────────────┘ \n"
+       << "   ┌─────── ▼ ────────────────────[\033[01;32md\033[0m]───────┐   \n"
+       << "   │              \033[01;36mDEVICE READY\033[0m              │   \n"
+       << "   └───────[\033[01;32mj\033[0m]──────────────────── ▲ ───────┘   \n"
+       << " ╔═════════ ▼ ═════════╗ ╔═════════╩══════════╗ \n"
+       << " ║  \033[01;33mINITIALIZING TASK\033[0m  ║ ║   \033[01;33mRESETTING TASK\033[0m   ║ \n"
+       << " ╚══════════╦══════════╝ ╚════════ ▲ ═════════╝ \n"
+       << "   ┌─────── ▼ ────────────────────[\033[01;32mt\033[0m]───────┐   \n"
+       << "   │                 \033[01;36mREADY\033[0m                  │   \n"
+       << "   └───────[\033[01;32mr\033[0m]──────────────────── ▲ ───────┘   \n"
+       << "    ╔══════ ▼ ════════════════════[\033[01;32ms\033[0m]══════╗    \n"
+       << "    ║               \033[01;33mRUNNING\033[0m                ║    \n"
+       << "    ╚══════════════════════════════════════╝    \n"
+       << "                                                \n";
+    cout << ss.str() << flush;
+}
+
+void Control::PrintStateMachine()
+{
+    stringstream ss;
+    ss << "               @                                                     \n"
+       << "   ┌───────────╨─────────────┐  ┌───────────┐                        \n"
+       << "   │           IDLE         [q]─▶  EXITING  │                        \n"
+       << "   └[i]─────────────────── ▲ ┘  └───────────┘                        \n"
+       << " ╔══ ▼ ════════════════╗ ╔═╩══════════════════╗                      \n"
+       << " ║ INITIALIZING DEVICE ║ ║  RESETTING DEVICE  ║                      \n"
+       << " ╚══════════╦══════════╝ ╚════════ ▲ ═════════╝                      \n"
+       << " ┌───────── ▼ ─────────┐           │  ┌────────────────────────────┐ \n"
+       << " │     INITIALIZED     │           │  │ Legend:                    │ \n"
+       << " └─────────[b]─────────┘           │  │----------------------------│ \n"
+       << " ╔═════════ ▼ ═════════╗           │  │ [k] keyboard shortcut for  │ \n"
+       << " ║       BINDING       ║           │  │     interactive controller │ \n"
+       << " ╚══════════╦══════════╝           │  │ ┌────────────────────────┐ │ \n"
+       << " ┌───────── ▼ ─────────┐           │  │ │      IDLING STATE      │ │ \n"
+       << " │        BOUND        │           │  │ └────────────────────────┘ │ \n"
+       << " └─────────[x]─────────┘           │  │ ╔════════════════════════╗ │ \n"
+       << " ╔═════════ ▼ ═════════╗           │  │ ║      WORKING STATE     ║ │ \n"
+       << " ║     CONNECTING      ║           │  │ ╚════════════════════════╝ │ \n"
+       << " ╚══════════╦══════════╝           │  └────────────────────────────┘ \n"
+       << "   ┌─────── ▼ ────────────────────[d]───────┐                        \n"
+       << "   │              DEVICE READY              │                        \n"
+       << "   └───────[j]──────────────────── ▲ ───────┘                        \n"
+       << " ╔═════════ ▼ ═════════╗ ╔═════════╩══════════╗                      \n"
+       << " ║  INITIALIZING TASK  ║ ║   RESETTING TASK   ║                      \n"
+       << " ╚══════════╦══════════╝ ╚════════ ▲ ═════════╝                      \n"
+       << "   ┌─────── ▼ ────────────────────[t]───────┐                        \n"
+       << "   │                 READY                  │                        \n"
+       << "   └───────[r]──────────────────── ▲ ───────┘                        \n"
+       << "    ╔══════ ▼ ════════════════════[s]══════╗                         \n"
+       << "    ║               RUNNING                ║                         \n"
+       << "    ╚══════════════════════════════════════╝                         \n"
+       << "                                                                     \n";
     cout << ss.str() << flush;
 }
 
 auto Control::WaitForNextState() -> DeviceState
 {
     unique_lock<mutex> lock{fEventsMutex};
-    while (fEvents.empty())
-    {
+    while (fEvents.empty()) {
         fNewEvent.wait_for(lock, chrono::milliseconds(50));
     }
 
     auto result = fEvents.front();
 
-    if (result == DeviceState::Error)
-    {
+    if (result == DeviceState::Error) {
         throw DeviceErrorState("Controlled device transitioned to error state.");
     }
 
@@ -351,14 +454,16 @@ auto Control::SignalHandler() -> void
 auto Control::RunShutdownSequence() -> void
 {
     auto nextState = GetCurrentDeviceState();
-    EmptyEventQueue();
-    while (nextState != DeviceState::Exiting && nextState != DeviceState::Error)
-    {
-        switch (nextState)
-        {
+    if (nextState != DeviceState::Error) {
+        EmptyEventQueue();
+    }
+    while (nextState != DeviceState::Exiting && nextState != DeviceState::Error) {
+        switch (nextState) {
             case DeviceState::Idle:
                 ChangeDeviceState(DeviceStateTransition::End);
                 break;
+            case DeviceState::Initialized:
+            case DeviceState::Bound:
             case DeviceState::DeviceReady:
                 ChangeDeviceState(DeviceStateTransition::ResetDevice);
                 break;
@@ -368,11 +473,8 @@ auto Control::RunShutdownSequence() -> void
             case DeviceState::Running:
                 ChangeDeviceState(DeviceStateTransition::Stop);
                 break;
-            case DeviceState::Paused:
-                ChangeDeviceState(DeviceStateTransition::Resume);
-                break;
             default:
-                LOG(debug) << "Controller ignoring event: " << nextState;
+                // LOG(debug) << "Controller ignoring event: " << nextState;
                 break;
         }
 
@@ -383,16 +485,6 @@ auto Control::RunShutdownSequence() -> void
     ReleaseDeviceControl();
 }
 
-auto Control::RunStartupSequence() -> void
-{
-    ChangeDeviceState(DeviceStateTransition::InitDevice);
-    while (WaitForNextState() != DeviceState::DeviceReady) {}
-    ChangeDeviceState(DeviceStateTransition::InitTask);
-    while (WaitForNextState() != DeviceState::Ready) {}
-    ChangeDeviceState(DeviceStateTransition::Run);
-    while (WaitForNextState() != DeviceState::Running) {}
-}
-
 auto Control::EmptyEventQueue() -> void
 {
     lock_guard<mutex> lock{fEventsMutex};

fairmq/plugins/Control.h

@@ -36,7 +36,10 @@ class Control : public Plugin
 
   private:
     auto InteractiveMode() -> void;
+    static auto PrintInteractiveHelpColor() -> void;
     static auto PrintInteractiveHelp() -> void;
+    static auto PrintStateMachineColor() -> void;
+    static auto PrintStateMachine() -> void;
     auto StaticMode() -> void;
     auto WaitForNextState() -> DeviceState;
     auto SignalHandler() -> void;
@@ -62,9 +65,7 @@ auto ControlPluginProgramOptions() -> Plugin::ProgOptions;
 REGISTER_FAIRMQ_PLUGIN(
     Control,   // Class name
     control,   // Plugin name (string, lower case chars only)
-    (Plugin::Version{FAIRMQ_VERSION_MAJOR,
-                     FAIRMQ_VERSION_MINOR,
-                     FAIRMQ_VERSION_PATCH}),       // Version
+    (Plugin::Version{FAIRMQ_VERSION_MAJOR, FAIRMQ_VERSION_MINOR, FAIRMQ_VERSION_PATCH}), // Version
     "FairRootGroup <fairroot@gsi.de>",             // Maintainer
     "https://github.com/FairRootGroup/FairRoot",   // Homepage
     ControlPluginProgramOptions   // Free function which declares custom program options for the

fairmq/plugins/DDS/DDS.cxx

@@ -40,7 +40,7 @@ DDS::DDS(const string& name, const Plugin::Version version, const string& mainta
     , fConnectingChans()
     , fStopMutex()
     , fStopCondition()
-    , fCommands({ "INIT DEVICE", "INIT TASK", "PAUSE", "RUN", "STOP", "RESET TASK", "RESET DEVICE" })
+    , fCommands({ "BIND", "CONNECT", "INIT TASK", "RUN", "STOP", "RESET TASK", "RESET DEVICE" })
     , fControllerThread()
     , fEvents()
     , fEventsMutex()
@@ -96,6 +96,10 @@ auto DDS::HandleControl() -> void
 
         ChangeDeviceState(DeviceStateTransition::InitDevice);
         while (WaitForNextState() != DeviceState::InitializingDevice) {}
+        ChangeDeviceState(DeviceStateTransition::CompleteInit);
+        while (WaitForNextState() != DeviceState::Initialized) {}
+        ChangeDeviceState(DeviceStateTransition::Bind);
+        while (WaitForNextState() != DeviceState::Bound) {}
 
         // in the Initializing state subscribe to receive addresses of connecting channels from DDS
         // and propagate addresses of bound channels to DDS.
@@ -114,6 +118,7 @@ auto DDS::HandleControl() -> void
         // publish bound addresses via DDS at keys corresponding to the channel prefixes, e.g. 'data' in data[i]
         PublishBoundChannels();
 
+        ChangeDeviceState(DeviceStateTransition::Connect);
         while (WaitForNextState() != DeviceState::DeviceReady) {}
 
         ChangeDeviceState(DeviceStateTransition::InitTask);
@@ -305,13 +310,26 @@ auto DDS::SubscribeForCustomCommands() -> void
 
         if (cmd == "check-state") {
             fDDSCustomCmd.send(id + ": " + ToStr(GetCurrentDeviceState()) + " (pid: " + pid + ")", to_string(senderId));
+        } else if (cmd == "INIT DEVICE") {
+            if (ChangeDeviceState(ToDeviceStateTransition(cmd))) {
+                fDDSCustomCmd.send(id + ": queued " + cmd + " transition", to_string(senderId));
+                while (WaitForNextState() != DeviceState::InitializingDevice) {}
+                ChangeDeviceState(DeviceStateTransition::CompleteInit);
+            } else {
+                fDDSCustomCmd.send(id + ": could not queue " + cmd + " transition", to_string(senderId));
+            }
         } else if (fCommands.find(cmd) != fCommands.end()) {
-            fDDSCustomCmd.send(id + ": attempting to " + cmd, to_string(senderId));
-            ChangeDeviceState(ToDeviceStateTransition(cmd));
+            if (ChangeDeviceState(ToDeviceStateTransition(cmd))) {
+                fDDSCustomCmd.send(id + ": queued " + cmd + " transition", to_string(senderId));
+            } else {
+                fDDSCustomCmd.send(id + ": could not queue " + cmd + " transition", to_string(senderId));
+            }
         } else if (cmd == "END") {
-            fDDSCustomCmd.send(id + ": attempting to " + cmd, to_string(senderId));
-            ChangeDeviceState(ToDeviceStateTransition(cmd));
-            fDDSCustomCmd.send(id + ": " + ToStr(GetCurrentDeviceState()), to_string(senderId));
+            if (ChangeDeviceState(ToDeviceStateTransition(cmd))) {
+                fDDSCustomCmd.send(id + ": queued " + cmd + " transition", to_string(senderId));
+            } else {
+                fDDSCustomCmd.send(id + ": could not queue " + cmd + " transition", to_string(senderId));
+            }
             if (ToStr(GetCurrentDeviceState()) == "EXITING") {
                 unique_lock<mutex> lock(fStopMutex);
                 fStopCondition.notify_one();
@@ -363,7 +381,7 @@ auto DDS::WaitForNextState() -> DeviceState
 {
     unique_lock<mutex> lock{fEventsMutex};
     while (fEvents.empty()) {
-        fNewEvent.wait(lock);
+        fNewEvent.wait_for(lock, chrono::milliseconds(50));
     }
 
     auto result = fEvents.front();

fairmq/plugins/DDS/runDDSCommandUI.cxx

@@ -25,7 +25,7 @@ namespace bpo = boost::program_options;
 void PrintControlsHelp()
 {
     cout << "Use keys to control the devices:" << endl;
-    cout << "[c] check states, [o] dump config, [h] help, [p] pause, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device" << endl;
+    cout << "[c] check states, [o] dump config, [h] help, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device, [b] bind, [x] connect" << endl;
     cout << "To quit press Ctrl+C" << endl;
 }
 
@@ -48,7 +48,7 @@ int main(int argc, char* argv[])
 
         if (vm.count("help")) {
             cout << "FairMQ DDS Command UI" << endl << options << endl;
-            cout << "Commands: [c] check state, [o] dump config, [h] help, [p] pause, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device" << endl;
+            cout << "Commands: [c] check state, [o] dump config, [h] help, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device" << endl;
             return EXIT_SUCCESS;
         }
 
@@ -96,14 +96,18 @@ int main(int argc, char* argv[])
                     cout << " > init devices" << endl;
                     ddsCustomCmd.send("INIT DEVICE", topologyPath);
                     break;
+                case 'b':
+                    cout << " > bind" << endl;
+                    ddsCustomCmd.send("BIND", topologyPath);
+                    break;
+                case 'x':
+                    cout << " > connect" << endl;
+                    ddsCustomCmd.send("CONNECT", topologyPath);
+                    break;
                 case 'j':
                     cout << " > init tasks" << endl;
                     ddsCustomCmd.send("INIT TASK", topologyPath);
                     break;
-                case 'p':
-                    cout << " > pause devices" << endl;
-                    ddsCustomCmd.send("PAUSE", topologyPath);
-                    break;
                 case 'r':
                     cout << " > run tasks" << endl;
                     ddsCustomCmd.send("RUN", topologyPath);

fairmq/plugins/PMIx/CMakeLists.txt

@@ -0,0 +1,27 @@
+################################################################################
+#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+set(plugin FairMQPlugin_pmix)
+add_library(${plugin} SHARED
+  ${CMAKE_CURRENT_SOURCE_DIR}/PMIxPlugin.cxx
+  ${CMAKE_CURRENT_SOURCE_DIR}/PMIxPlugin.h
+  ${CMAKE_CURRENT_SOURCE_DIR}/PMIx.hpp
+)
+target_link_libraries(${plugin} FairMQ PMIx::libpmix)
+target_include_directories(${plugin} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
+set_target_properties(${plugin} PROPERTIES
+  CXX_VISIBILITY_PRESET hidden
+  VERSION ${PROJECT_GIT_VERSION}
+  SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
+)
+
+install(TARGETS ${plugin}
+  EXPORT ${PROJECT_EXPORT_SET}
+  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
+  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
+)

fairmq/plugins/PMIx/PMIx.hpp

@@ -0,0 +1,219 @@
+/********************************************************************************
+ *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef PMIX_HPP
+#define PMIX_HPP
+
+#include <cstring>
+#include <limits>
+#include <memory>
+#include <ostream>
+#include <pmix.h>
+#include <stdexcept>
+#include <string.h>
+#include <type_traits>
+#include <utility>
+#include <vector>
+#include <FairMQLogger.h>
+
+// C++ PMIx v2.1 API
+namespace pmix
+{
+
+struct runtime_error : std::runtime_error
+{
+    using std::runtime_error::runtime_error;
+};
+
+using status = pmix_status_t;
+
+using nspace = pmix_nspace_t;
+
+using key = pmix_key_t;
+
+using data_type = pmix_data_type_t;
+
+struct rank
+{
+    enum named : pmix_rank_t
+    {
+        undef = PMIX_RANK_UNDEF,
+        wildcard = PMIX_RANK_WILDCARD,
+        local_node = PMIX_RANK_LOCAL_NODE
+    };
+
+    explicit rank(pmix_rank_t r)
+        : m_value(r)
+    {}
+
+    operator pmix_rank_t() { return m_value; }
+
+  private:
+    pmix_rank_t m_value;
+};
+
+struct proc : pmix_proc_t
+{
+    proc() { PMIX_PROC_CONSTRUCT(static_cast<pmix_proc_t*>(this)); }
+    ~proc() { PMIX_PROC_DESTRUCT(static_cast<pmix_proc_t*>(this)); }
+
+    proc(pmix::nspace ns, pmix::rank r)
+    {
+        PMIX_PROC_LOAD(static_cast<pmix_proc_t*>(this), ns, static_cast<pmix_rank_t>(r));
+    }
+
+    friend std::ostream& operator<<(std::ostream& os, const proc& p)
+    {
+        return os << "nspace=" << p.nspace << ",rank=" << p.rank;
+    }
+};
+
+struct value : pmix_value_t
+{
+    value() { PMIX_VALUE_CONSTRUCT(static_cast<pmix_value_t*>(this)); }
+    ~value() { PMIX_VALUE_DESTRUCT(static_cast<pmix_value_t*>(this)); }
+
+    value(const value& rhs)
+    {
+      LOG(warn) << "copy ctor";
+        status rc;
+        auto lhs(static_cast<pmix_value_t*>(this));
+        PMIX_VALUE_XFER(rc, lhs, static_cast<pmix_value_t*>(const_cast<value*>(&rhs)));
+
+        if (rc != PMIX_SUCCESS) {
+            throw runtime_error("pmix::value copy ctor failed: rc=" + rc);
+        }
+    }
+
+    // template<typename T>
+    // value(const T* val, data_type dt)
+    // {
+        // PMIX_VALUE_LOAD(static_cast<pmix_value_t*>(this), const_cast<void*>(val), dt);
+    // }
+
+    template<typename T>
+    explicit value(T)
+    {
+        throw runtime_error("Given value type not supported or not yet implemented.");
+    }
+
+    explicit value(const char* val)
+    {
+        PMIX_VALUE_LOAD(static_cast<pmix_value_t*>(this), const_cast<char*>(val), PMIX_STRING);
+    }
+
+    explicit value(const std::string& val)
+    {
+        PMIX_VALUE_LOAD(
+            static_cast<pmix_value_t*>(this), const_cast<char*>(val.c_str()), PMIX_STRING);
+    }
+};
+
+struct info : pmix_info_t
+{
+    info() { PMIX_INFO_CONSTRUCT(static_cast<pmix_info_t*>(this)); }
+    ~info() { PMIX_INFO_DESTRUCT(static_cast<pmix_info_t*>(this)); }
+
+    template<typename... Args>
+    info(const std::string& k, Args&&... args)
+    {
+        (void)strncpy(key, k.c_str(), PMIX_MAX_KEYLEN);
+        flags = 0;
+
+        pmix::value rhs(std::forward<Args>(args)...);
+        auto lhs(&value);
+        status rc;
+        PMIX_VALUE_XFER(rc, lhs, static_cast<pmix_value_t*>(&rhs));
+
+        if (rc != PMIX_SUCCESS) {
+            throw runtime_error("pmix::info ctor failed: rc=" + std::to_string(rc));
+        }
+    }
+
+    friend std::ostream& operator<<(std::ostream& os, const info& p)
+    {
+        return os << "key=" << p.key << ",value='" << p.value.data.string << "'";
+    }
+};
+
+struct pdata : pmix_pdata_t
+{
+    pdata() { PMIX_PDATA_CONSTRUCT(static_cast<pmix_pdata_t*>(this)); }
+    ~pdata() { PMIX_PDATA_DESTRUCT(static_cast<pmix_pdata_t*>(this)); }
+    pdata(const pdata& rhs)
+    {
+        PMIX_PDATA_XFER(static_cast<pmix_pdata_t*>(this),
+                        static_cast<pmix_pdata_t*>(const_cast<pdata*>(&rhs)));
+    }
+
+    auto set_key(const std::string& new_key) -> void
+    {
+        (void)strncpy(key, new_key.c_str(), PMIX_MAX_KEYLEN);
+    }
+};
+
+auto init(const std::vector<info>& info = {}) -> proc
+{
+    proc res;
+    status rc;
+
+    rc = PMIx_Init(&res, const_cast<pmix::info*>(info.data()), info.size());
+    if (rc != PMIX_SUCCESS) {
+        throw runtime_error("pmix::init() failed: rc=" + std::to_string(rc));
+    }
+
+    return res;
+}
+
+auto initialized() -> bool { return !!PMIx_Initialized(); }
+
+auto get_version() -> const char* { return PMIx_Get_version(); }
+
+auto finalize(const std::vector<info>& info = {}) -> void
+{
+    status rc;
+
+    rc = PMIx_Finalize(info.data(), info.size());
+    if (rc != PMIX_SUCCESS) {
+        throw runtime_error("pmix::finalize() failed: rc=" + std::to_string(rc));
+    }
+}
+
+auto publish(const std::vector<info>& info) -> void
+{
+    status rc;
+
+    rc = PMIx_Publish(info.data(), info.size());
+    if (rc != PMIX_SUCCESS) {
+        throw runtime_error("pmix::publish() failed: rc=" + std::to_string(rc));
+    }
+}
+
+auto fence(const std::vector<proc>& procs = {}, const std::vector<info>& info = {}) -> void
+{
+    status rc;
+
+    rc = PMIx_Fence(procs.data(), procs.size(), info.data(), info.size());
+    if (rc != PMIX_SUCCESS) {
+        throw runtime_error("pmix::fence() failed: rc=" + std::to_string(rc));
+    }
+}
+
+auto lookup(std::vector<pdata>& pdata, const std::vector<info>& info = {}) -> void
+{
+    status rc;
+
+    rc = PMIx_Lookup(pdata.data(), pdata.size(), info.data(), info.size());
+    if (rc != PMIX_SUCCESS) {
+        throw runtime_error("pmix::lookup() failed: rc=" + std::to_string(rc));
+    }
+}
+
+} /* namespace pmix */
+
+#endif /* PMIX_HPP */

fairmq/plugins/PMIx/PMIxPlugin.cxx

@@ -0,0 +1,146 @@
+/********************************************************************************
+ *    Copyright (C) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "PMIxPlugin.h"
+
+#include <boost/algorithm/string/join.hpp>
+#include <fairmq/Tools.h>
+#include <stdexcept>
+
+namespace fair
+{
+namespace mq
+{
+namespace plugins
+{
+
+PMIxPlugin::PMIxPlugin(const std::string& name,
+                       const Plugin::Version version,
+                       const std::string& maintainer,
+                       const std::string& homepage,
+                       PluginServices* pluginServices)
+    : Plugin(name, version, maintainer, homepage, pluginServices)
+    , fPid(getpid())
+{
+    SubscribeToDeviceStateChange([&](DeviceState newState) {
+        switch (newState) {
+            case DeviceState::Connecting:
+                Init();
+                Publish();
+                Fence();
+                Lookup();
+                break;
+        case DeviceState::Exiting:
+            UnsubscribeFromDeviceStateChange();
+            break;
+        default:
+            break;
+        }
+    });
+}
+
+PMIxPlugin::~PMIxPlugin()
+{
+    while (pmix::initialized()) {
+        try {
+            pmix::finalize();
+            LOG(debug) << PMIxClient() << " pmix::finalize() OK";
+        } catch (const pmix::runtime_error& e) {
+            LOG(debug) << PMIxClient() << " pmix::finalize() failed: " << e.what();
+        }
+    }
+}
+
+auto PMIxPlugin::PMIxClient() const -> std::string
+{
+    std::stringstream ss;
+    ss << "PMIx client(pid=" << fPid << ")";
+    return ss.str();
+}
+
+auto PMIxPlugin::Init() -> void
+{
+    if (!pmix::initialized()) {
+        fProc = pmix::init();
+        LOG(debug) << PMIxClient() << " pmix::init() OK: " << fProc
+                   << ",version=" << pmix::get_version();
+    }
+}
+
+auto PMIxPlugin::Publish() -> void
+{
+    auto channels(GetChannelInfo());
+    std::vector<pmix::info> info;
+
+    for (const auto& c : channels) {
+        std::string methodKey{"chans." + c.first + "." + std::to_string(c.second - 1) + ".method"};
+        if (GetProperty<std::string>(methodKey) == "bind") {
+            for (int i = 0; i < c.second; ++i) {
+                std::string addressKey{"chans." + c.first + "." + std::to_string(i) + ".address"};
+                info.emplace_back(addressKey, GetProperty<std::string>(addressKey));
+                LOG(debug) << PMIxClient() << info.back();
+            }
+        }
+    }
+
+    if (info.size() > 0) {
+        pmix::publish(info);
+        LOG(debug) << PMIxClient() << " pmix::publish() OK: published "
+                   << info.size() << " binding channels.";
+    }
+}
+
+auto PMIxPlugin::Fence() -> void
+{
+    pmix::proc all(fProc);
+    all.rank = pmix::rank::wildcard;
+
+    pmix::fence({all});
+}
+
+auto PMIxPlugin::Lookup() -> void
+{
+    auto channels(GetChannelInfo());
+    std::vector<pmix::pdata> pdata;
+
+    for (const auto& c : channels) {
+        std::string methodKey{"chans." + c.first + "." + std::to_string(c.second - 1) + ".method"};
+        if (GetProperty<std::string>(methodKey) == "connect") {
+            for (int i = 0; i < c.second; ++i) {
+                std::string addressKey{"chans." + c.first + "." + std::to_string(i) + ".address"};
+                pdata.emplace_back();
+                pdata.back().set_key(addressKey);
+            }
+        }
+    }
+
+    if (pdata.size() > 0) {
+        pmix::lookup(pdata);
+        LOG(debug) << PMIxClient() << " pmix::lookup() OK";
+    }
+
+    LOG(info) << pdata.size();
+
+    for (const auto& p : pdata) {
+        if (p.value.type == PMIX_UNDEF) {
+            LOG(debug) << PMIxClient() << " pmix::lookup() not found: key=" << p.key;
+        } else if (p.value.type == PMIX_STRING) {
+            LOG(debug) << PMIxClient() << " pmix::lookup() found: key=" << p.key << ",value=" << p.value.data.string;
+            SetProperty<std::string>(p.key, p.value.data.string);
+            LOG(info) << GetProperty<std::string>(p.key);
+        } else {
+            LOG(debug) << PMIxClient() << " pmix::lookup() wrong type returned: key=" << p.key << ",type=" << p.value.type;
+        }
+    }
+
+    LOG(info) << pdata.size();
+}
+
+} /* namespace plugins */
+} /* namespace mq */
+} /* namespace fair */

fairmq/plugins/PMIx/PMIxPlugin.h

@@ -0,0 +1,77 @@
+/********************************************************************************
+ *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_PLUGINS_PMIX
+#define FAIR_MQ_PLUGINS_PMIX
+
+#include "PMIx.hpp"
+
+#include <fairmq/Plugin.h>
+#include <fairmq/Version.h>
+#include <FairMQLogger.h>
+
+#include <string>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <sys/types.h>
+#include <unistd.h>
+#include <vector>
+
+namespace fair
+{
+namespace mq
+{
+namespace plugins
+{
+
+class PMIxPlugin : public Plugin
+{
+  public:
+    PMIxPlugin(const std::string& name,
+         const Plugin::Version version,
+         const std::string& maintainer,
+         const std::string& homepage,
+         PluginServices* pluginServices);
+    ~PMIxPlugin();
+    auto PMIxClient() const -> std::string;
+
+  private:
+    pmix::proc fProc;
+    pid_t fPid;
+
+    auto Init() -> void;
+    auto Publish() -> void;
+    auto Fence() -> void;
+    auto Lookup() -> void;
+};
+
+Plugin::ProgOptions PMIxProgramOptions()
+{
+    boost::program_options::options_description options{"PMIx Plugin"};
+    options.add_options()(
+        "pmix-dummy", boost::program_options::value<int>()->default_value(0), "Dummy.");
+    return options;
+}
+
+REGISTER_FAIRMQ_PLUGIN(
+    PMIxPlugin,                                  // Class name
+    pmix,                                        // Plugin name (string, lower case chars only)
+    (Plugin::Version{FAIRMQ_VERSION_MAJOR,
+                     FAIRMQ_VERSION_MINOR,
+                     FAIRMQ_VERSION_PATCH}),     // Version
+    "FairRootGroup <fairroot@gsi.de>",           // Maintainer
+    "https://github.com/FairRootGroup/FairMQ",   // Homepage
+    PMIxProgramOptions                           // custom program options for the plugin
+)
+
+} /* namespace plugins */
+} /* namespace mq */
+} /* namespace fair */
+
+#endif /* FAIR_MQ_PLUGINS_PMIX */

fairmq/run/startMQBenchmark.sh.in

@@ -61,7 +61,7 @@ SAMPLER+=" --msg-size $msgSize"
 SAMPLER+=" --num-parts 1"
 # SAMPLER+=" --msg-rate 1000"
 SAMPLER+=" --max-iterations $maxIterations"
-SAMPLER+=" --channel-config name=data,type=push,method=bind,address=tcp://127.0.0.1:5555"
+SAMPLER+=" --channel-config name=data,type=pair,method=bind,address=tcp://127.0.0.1:5555"
 xterm -geometry 90x50+0+0 -hold -e $affinitySamp @CMAKE_CURRENT_BINARY_DIR@/$SAMPLER &
 echo ""
 echo "started: xterm -geometry 90x50+0+0 -hold -e $affinitySamp @CMAKE_CURRENT_BINARY_DIR@/$SAMPLER"
@@ -72,10 +72,11 @@ SINK+=" --id sink1"
 #SINK+=" --io-threads 2"
 #SINK+=" --control static"
 SINK+=" --transport $transport"
+SINK+=" --ofi-size-hint $msgSize"
 SINK+=" --severity debug"
 SINK+=" --multipart false"
 SINK+=" --max-iterations $maxIterations"
-SINK+=" --channel-config name=data,type=pull,method=connect,address=tcp://127.0.0.1:5555"
+SINK+=" --channel-config name=data,type=pair,method=connect,address=tcp://127.0.0.1:5555"
 xterm -geometry 90x50+550+0 -hold -e $affinitySink @CMAKE_CURRENT_BINARY_DIR@/$SINK &
 echo ""
 echo "started: xterm -geometry 90x50+550+0 -hold -e $affinitySink @CMAKE_CURRENT_BINARY_DIR@/$SINK"

fairmq/shmem/FairMQMessageSHM.cxx

@@ -350,6 +350,7 @@ void FairMQMessageSHM::CloseMessage()
                 }
                 if (fRegionPtr)
                 {
+                    // LOG(debug) << "sending ack";
                     if (fRegionPtr->fQueue->timed_send(&block, sizeof(RegionBlock), 0, sndTill))
                     {
                         success = true;

fairmq/shmem/FairMQPollerSHM.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**

fairmq/zeromq/FairMQPollerZMQ.cxx

@@ -1,8 +1,8 @@
 /********************************************************************************
  *    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
  *                                                                              *
- *              This software is distributed under the terms of the             * 
- *              GNU Lesser General Public Licence (LGPL) version 3,             *  
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 /**