Fix incorrect parameters in region example scripts

Apparently:

  "warning: unqualified call to 'std::move' [-Wunqualified-std-cast-call]"

is default in new XCode.

.github/workflows/codemeta_validate.yaml

@@ -0,0 +1,21 @@
+name: validate codemeta
+
+on:
+  push:
+    paths:
+    - codemeta.json
+    - .github/workflows/codemeta_validate.yaml
+  pull_request:
+    paths:
+    - codemeta.json
+    - .github/workflows/codemeta_validate.yaml
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    container:
+      image: gitlab-registry.in2p3.fr/escape2020/wp3/eossr:v1.0
+    steps:
+      - uses: actions/checkout@v3
+      - name: validate codemeta
+        run: eossr-metadata-validator codemeta.json

.github/workflows/fair-software.yml

@@ -0,0 +1,16 @@
+name: fair-software
+
+on: push
+
+jobs:
+  verify:
+    name: "fair-software"
+    runs-on: ubuntu-latest
+    if: github.repository == 'FairRootGroup/FairMQ'
+    steps:
+      - uses: fair-software/howfairis-github-action@0.2.1
+        name: Measure compliance with fair-software.eu recommendations
+        env:
+          PYCHARM_HOSTED: "Trick colorama into displaying colored output"
+        with:
+          MY_REPO_URL: "https://github.com/${{ github.repository }}"

.gitignore

@@ -4,3 +4,5 @@ install
 .vscode
 /compile_commands.json
 .cache
+.spack-env
+spack.lock

.zenodo.json

@@ -0,0 +1,88 @@
+{
+    "creators": [
+        {
+            "orcid": "0000-0002-8071-4497",
+            "name": "Al-Turany, Mohammad"
+        },
+        {
+            "orcid": "0000-0003-3787-1910",
+            "name": "Klein, Dennis"
+        },
+        {
+            "name": "Kollegger, Thorsten"
+        },
+        {
+            "orcid": "0000-0002-6249-155X",
+            "name": "Rybalchenko, Alexey"
+        },
+        {
+            "name": "Winckler, Nicolas"
+        }
+    ],
+    "contributors": [
+        {
+            "type": "Other",
+            "name": "Aphecetche, Laurent"
+        },
+        {
+            "type": "Other",
+            "name": "Binet, Sebastien"
+        },
+        {
+            "type": "Other",
+            "name": "Eulisse, Giulio"
+        },
+        {
+            "type": "Other",
+            "name": "Karabowicz, Radoslaw"
+        },
+        {
+            "type": "Other",
+            "name": "Kretz, Matthias"
+        },
+        {
+            "type": "Other",
+            "name": "Krzewicki, Mikolaj"
+        },
+        {
+            "type": "Other",
+            "name": "Lebedev, Andrey"
+        },
+        {
+            "type": "Other",
+            "name": "Mrnjavac, Teo"
+        },
+        {
+            "type": "Other",
+            "name": "Neskovic, Gvozden"
+        },
+        {
+            "type": "Other",
+            "name": "Richter, Matthias"
+        },
+        {
+            "type": "Other",
+            "orcid": "0000-0002-5321-8404",
+            "name": "Tacke, Christian"
+        },
+        {
+            "type": "Other",
+            "name": "Uhlig, Florian"
+        },
+        {
+            "type": "Other",
+            "name": "Wenzel, Sandro"
+        }
+    ],
+    "description": "<p>C++ Message Queuing Library and Framework</p>",
+    "related_identifiers": [
+        {
+            "identifier": "https://github.com/FairRootGroup/FairMQ/",
+            "relation": "isSupplementTo",
+            "resource_type": "software",
+            "scheme": "url"
+        }
+    ],
+    "title": "FairMQ",
+    "license": "LGPL-3.0-only"
+}

AUTHORS

@@ -1,5 +1,5 @@
-Al-Turany, Mohammad
-Klein, Dennis
+Al-Turany, Mohammad [https://orcid.org/0000-0002-8071-4497]
+Klein, Dennis [https://orcid.org/0000-0003-3787-1910]
 Kollegger, Thorsten
-Rybalchenko, Alexey
+Rybalchenko, Alexey [https://orcid.org/0000-0002-6249-155X]
 Winckler, Nicolas

CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (C) 2018-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
+# Copyright (C) 2018-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
 #                                                                              #
 #              This software is distributed under the terms of the             #
 #              GNU Lesser General Public Licence (LGPL) version 3,             #
@@ -9,7 +9,7 @@
 
 # Project ######################################################################
 cmake_minimum_required(VERSION 3.15 FATAL_ERROR)
-cmake_policy(VERSION 3.15...3.22)
+cmake_policy(VERSION 3.15...3.26)
 
 list(PREPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
 include(GitHelper)
@@ -27,10 +27,6 @@ fairmq_build_option(BUILD_FAIRMQ        "Build FairMQ library and devices."
                                          DEFAULT ON)
 fairmq_build_option(BUILD_TESTING       "Build tests."
                                          DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
-fairmq_build_option(BUILD_OFI_TRANSPORT "Build experimental OFI transport."
-                                         DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
-fairmq_build_option(BUILD_PMIX_PLUGIN   "Build PMIx plugin."
-                                         DEFAULT OFF REQUIRES "BUILD_FAIRMQ")
 fairmq_build_option(BUILD_EXAMPLES      "Build FairMQ examples."
                                          DEFAULT ON  REQUIRES "BUILD_FAIRMQ")
 fairmq_build_option(BUILD_TIDY_TOOL     "Build the fairmq-tidy tool."
@@ -86,12 +82,6 @@ endif()
 if(BUILD_TESTING)
   list(APPEND PROJECT_PACKAGE_COMPONENTS tests)
 endif()
-if(BUILD_PMIX_PLUGIN)
-  list(APPEND PROJECT_PACKAGE_COMPONENTS pmix_plugin)
-endif()
-if(BUILD_OFI_TRANSPORT)
-  list(APPEND PROJECT_PACKAGE_COMPONENTS ofi_transport)
-endif()
 if(BUILD_EXAMPLES)
   list(APPEND PROJECT_PACKAGE_COMPONENTS examples)
 endif()
@@ -137,5 +127,6 @@ fairmq_summary_components()
 fairmq_summary_static_analysis()
 fairmq_summary_install_prefix()
 fairmq_summary_debug_mode()
+fairmq_summary_compile_definitions()
 message(STATUS "  ")
 ################################################################################

CONTRIBUTORS

@@ -3,11 +3,11 @@ Binet, Sebastien
 Eulisse, Giulio
 Karabowicz, Radoslaw
 Kretz, Matthias <kretz@kde.org>
-Krzewicki, Mikolaj 
+Krzewicki, Mikolaj
 Lebedev, Andrey
 Mrnjavac, Teo <teo.m@cern.ch>
 Neskovic, Gvozden
 Richter, Matthias
-Tacke, Christian
+Tacke, Christian [https://orcid.org/0000-0002-5321-8404]
 Uhlig, Florian
 Wenzel, Sandro

FairMQTest.cmake

@@ -1,5 +1,5 @@
 ################################################################################
-#    Copyright (C) 2021 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+# Copyright (C) 2021-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
 #                                                                              #
 #              This software is distributed under the terms of the             #
 #              GNU Lesser General Public Licence (LGPL) version 3,             #
@@ -42,12 +42,6 @@ endif()
 ctest_start(Continuous)
 
 list(APPEND options "-DDISABLE_COLOR=ON" "-DBUILD_EXAMPLES=ON" "-DBUILD_TESTING=ON")
-if(HAS_PMIX)
-  list(APPEND options "-DBUILD_PMIX_PLUGIN=ON")
-endif()
-if(HAS_ASIO AND HAS_ASIOFI)
-  list(APPEND options "-DBUILD_OFI_TRANSPORT=ON")
-endif()
 if(RUN_STATIC_ANALYSIS)
   list(APPEND options "-DRUN_STATIC_ANALYSIS=ON")
 endif()
@@ -87,7 +81,7 @@ ctest_submit()
 
 if(NOT RUN_STATIC_ANALYSIS)
   ctest_test(BUILD "${CTEST_BINARY_DIRECTORY}"
-             PARALLEL_LEVEL 1
+             PARALLEL_LEVEL ${NCPUS}
              SCHEDULE_RANDOM ON
              RETURN_VALUE _ctest_test_ret_val)
 

Jenkinsfile

@@ -4,20 +4,21 @@ def jobMatrix(String type, List specs) {
   def nodes = [:]
   for (spec in specs) {
     def job = ""
-    def selector = ""
+    def selector = "slurm"
     def os = ""
     def ver = ""
 
     if (type == 'build') {
       job = "${spec.os}-${spec.ver}-${spec.arch}-${spec.compiler}"
-      selector = "${spec.os}-${spec.ver}-${spec.arch}"
+      if (spec.os =~ /^macos/) {
+        selector = "${spec.os}-${spec.ver}-${spec.arch}"
+      }
       os = spec.os
       ver = spec.ver
     } else { // == 'check'
       job = "${spec.name}"
-      selector = 'fedora-35-x86_64'
       os = 'fedora'
-      ver = '35'
+      ver = '36'
     }
 
     def label = "${job}"
@@ -36,14 +37,14 @@ def jobMatrix(String type, List specs) {
           sh "echo \"export LABEL=\\\"\${JOB_BASE_NAME} ${label}\\\"\" >> ${jobscript}"
           if (selector =~ /^macos/) {
             sh """\
-              echo \"export DDS_ROOT=\\\"\\\$(brew --prefix dds)\\\"\" >> ${jobscript}
-              echo \"export PATH=\\\"\\\$(brew --prefix dds)/bin:\\\$PATH\\\"\" >> ${jobscript}
               echo \"${ctestcmd}\" >> ${jobscript}
             """
             sh "cat ${jobscript}"
             sh "bash ${jobscript}"
-          } else {
-            def containercmd = "singularity exec --net --ipc --uts --pid -B/shared ${env.SINGULARITY_CONTAINER_ROOT}/fairmq/${os}.${ver}.sif bash -l -c \\\"${ctestcmd} ${extra}\\\""
+          } else { // selector == "slurm"
+            def imageurl = "oras://ghcr.io/fairrootgroup/fairmq-dev/${os}-${ver}-sif:latest"
+            def execopts = "--net --ipc --uts --pid -B/shared"
+            def containercmd = "singularity exec ${execopts} ${imageurl} bash -l -c \\\"${ctestcmd} ${extra}\\\""
             sh """\
               echo \"echo \\\"*** Job started at .......: \\\$(date -R)\\\"\" >> ${jobscript}
               echo \"echo \\\"*** Job ID ...............: \\\${SLURM_JOB_ID}\\\"\" >> ${jobscript}
@@ -67,12 +68,6 @@ def jobMatrix(String type, List specs) {
           deleteDir()
           githubNotify(context: "${label}", description: 'Success', status: 'SUCCESS')
         } catch (e) {
-          def tarball = "${type}_${job}_dds_logs.tar.gz"
-          if (fileExists("build/test/.DDS")) {
-            sh "tar czvf ${tarball} -C \${WORKSPACE}/build/test .DDS/"
-            archiveArtifacts tarball
-          }
-
           deleteDir()
           githubNotify(context: "${label}", description: 'Error', status: 'ERROR')
           throw e
@@ -89,19 +84,20 @@ pipeline{
     stage("CI") {
       steps{
         script {
-          def all = '-DHAS_ASIO=ON -DHAS_ASIOFI=ON -DHAS_PMIX=ON'
-
           def builds = jobMatrix('build', [
-            [os: 'ubuntu', ver: '20.04', arch: 'x86_64', compiler: 'gcc-9',  extra: all],
-            [os: 'fedora', ver: '32',    arch: 'x86_64', compiler: 'gcc-10', extra: all],
-            [os: 'fedora', ver: '33',    arch: 'x86_64', compiler: 'gcc-10', extra: all],
-            [os: 'fedora', ver: '34',    arch: 'x86_64', compiler: 'gcc-11', extra: all],
-            [os: 'fedora', ver: '35',    arch: 'x86_64', compiler: 'gcc-11', extra: all],
-            [os: 'macos',  ver: '12',    arch: 'x86_64', compiler: 'apple-clang-13', extra: '-DHAS_ASIO=ON'],
-            [os: 'macos',  ver: '12',    arch: 'arm64', compiler: 'apple-clang-13', extra: '-DHAS_ASIO=ON'],
+            [os: 'ubuntu', ver: '20.04', arch: 'x86_64', compiler: 'gcc-9'],
+            [os: 'ubuntu', ver: '22.04', arch: 'x86_64', compiler: 'gcc-11'],
+            [os: 'fedora', ver: '33',    arch: 'x86_64', compiler: 'gcc-10'],
+            [os: 'fedora', ver: '34',    arch: 'x86_64', compiler: 'gcc-11'],
+            [os: 'fedora', ver: '35',    arch: 'x86_64', compiler: 'gcc-11'],
+            [os: 'fedora', ver: '36',    arch: 'x86_64', compiler: 'gcc-12'],
+            [os: 'fedora', ver: '37',    arch: 'x86_64', compiler: 'gcc-12'],
+            [os: 'fedora', ver: '38',    arch: 'x86_64', compiler: 'gcc-13'],
+            [os: 'macos',  ver: '12',    arch: 'x86_64', compiler: 'apple-clang-14'],
+            [os: 'macos',  ver: '13',    arch: 'arm64',  compiler: 'apple-clang-14'],
           ])
 
-          def all_debug = "${all} -DCMAKE_BUILD_TYPE=Debug"
+          def all_debug = "-DCMAKE_BUILD_TYPE=Debug"
 
           def checks = jobMatrix('check', [
             [name: 'static-analyzers', extra: "${all_debug} -DRUN_STATIC_ANALYSIS=ON"],

README.md

@@ -1,5 +1,11 @@
 <!-- {#mainpage} -->
-# FairMQ [![license](https://alfa-ci.gsi.de/shields/badge/license-LGPL--3.0-orange.svg)](COPYRIGHT)
+# FairMQ
+
+[![license](https://alfa-ci.gsi.de/shields/badge/license-LGPL--3.0-orange.svg)](COPYRIGHT)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1689985.svg)](https://doi.org/10.5281/zenodo.1689985)
+[![OpenSSF Best Practices](https://bestpractices.coreinfrastructure.org/projects/6915/badge)](https://bestpractices.coreinfrastructure.org/projects/6915)
+[![fair-software.eu](https://img.shields.io/badge/fair--software.eu-%E2%97%8F%20%20%E2%97%8F%20%20%E2%97%8B%20%20%E2%97%8F%20%20%E2%97%8F-yellow)](https://github.com/FairRootGroup/FairMQ/actions/workflows/fair-software.yml)
+[![Spack package](https://repology.org/badge/version-for-repo/spack/fairmq.svg)](https://repology.org/project/fairmq/versions)
 
 C++ Message Queuing Library and Framework
 
@@ -18,7 +24,8 @@ FairMQ is designed to help implementing large-scale data processing workflows ne
 The core of FairMQ provides an abstract asynchronous message passing API with scalability protocols
 inspired by [ZeroMQ](https://github.com/zeromq/libzmq) (e.g. PUSH/PULL, PUB/SUB).
 FairMQ provides multiple implementations for its API (so-called "transports",
-e.g. `zeromq`, `shmem` and `ofi` (in development)) to cover a variety of use cases
+e.g. `zeromq` and `shmem` (latest release of the `ofi` transport in v1.4.56, removed since v1.5+)) to cover
+a variety of use cases
 (e.g. inter-thread, inter-process, inter-node communication) and machines (e.g. Ethernet, Infiniband).
 In addition to this core functionality FairMQ provides a framework for creating "devices" - actors which
 are communicating through message passing. FairMQ does not only allow the user to use different transport
@@ -38,10 +45,10 @@ Recommended:
 
 ```bash
 git clone https://github.com/FairRootGroup/FairMQ fairmq_source
-cmake -S fairmq_source -B fairmq_build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=fairmq_install
+cmake -S fairmq_source -B fairmq_build -GNinja -DCMAKE_BUILD_TYPE=Release
 cmake --build fairmq_build
-cmake --build fairmq_build --target test
-cmake --build fairmq_build --target install
+ctest --test-dir fairmq_build --output-on-failure --schedule-random -j<ncpus>
+cmake --install fairmq_build --prefix $(pwd)/fairmq_install
 ```
 
 Please consult the [manpages of your CMake version](https://cmake.org/cmake/help/latest/manual/cmake.1.html) for more options.
@@ -49,6 +56,7 @@ Please consult the [manpages of your CMake version](https://cmake.org/cmake/help
 If dependencies are not installed in standard system directories, you can hint the installation location via
 `-DCMAKE_PREFIX_PATH=...` or per dependency via `-D{DEPENDENCY}_ROOT=...` (`*_ROOT` variables can also be environment variables).
 
+
 ## Usage
 
 FairMQ ships as a CMake package, so in your `CMakeLists.txt` you can discover it like this:
@@ -68,25 +76,14 @@ If FairMQ is not installed in system directories, you can hint the installation:
 list(PREPEND CMAKE_PREFIX_PATH /path/to/fairmq_install)
 ```
 
-Optionally, you can require certain FairMQ package components and a minimum version:
-
-```cmake
-find_package(FairMQ 1.4.50 COMPONENTS ofi_transport)
-```
-
-When building FairMQ, CMake will print a summary table of all available package components.
-
 ## Dependencies
 
-  * [asio](https://github.com/chriskohlhoff/asio)
-  * [asiofi](https://github.com/FairRootGroup/asiofi)
   * [Boost](https://www.boost.org/)
   * [CMake](https://cmake.org/)
   * [Doxygen](http://www.doxygen.org/)
   * [FairCMakeModules](https://github.com/FairRootGroup/FairCMakeModules) (optionally bundled)
   * [FairLogger](https://github.com/FairRootGroup/FairLogger)
   * [GTest](https://github.com/google/googletest) (optionally bundled)
-  * [PMIx](https://pmix.org/)
   * [ZeroMQ](http://zeromq.org/)
 
   Which dependencies are required depends on which components are built.
@@ -100,9 +97,8 @@ On command line:
   * `-DDISABLE_COLOR=ON` disables coloured console output.
   * `-DBUILD_TESTING=OFF` disables building of tests.
   * `-DBUILD_EXAMPLES=OFF` disables building of examples.
-  * `-DBUILD_OFI_TRANSPORT=ON` enables building of the experimental OFI transport.
-  * `-DBUILD_PMIX_PLUGIN=ON` enables building of the PMIx plugin.
   * `-DBUILD_DOCS=ON` enables building of API docs.
+  * `-DFAIRMQ_CHANNEL_DEFAULT_AUTOBIND=OFF` disable channel `autoBind` by default
   * You can hint non-system installations for dependent packages, see the #installation-from-source section above
 
 After the `find_package(FairMQ)` call the following CMake variables are defined:
@@ -159,4 +155,4 @@ After the `find_package(FairMQ)` call the following CMake variables are defined:
    1. [Usage](docs/Plugins.md#71-usage)
    2. [Development](docs/Plugins.md#72-development)
    3. [Provided Plugins](docs/Plugins.md#73-provided-plugins)
-       2. [PMIx](docs/Plugins.md#731-pmix)
+       1. [PMIx](docs/Plugins.md#731-pmix)

cmake/FairMQDependencies.cmake

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (C) 2018-2021 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
+# Copyright (C) 2018-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
 #                                                                              #
 #              This software is distributed under the terms of the             #
 #              GNU Lesser General Public Licence (LGPL) version 3,             #
@@ -18,15 +18,6 @@ if(BUILD_FAIRMQ)
   set(Threads_PREFIX "<system>")
 endif()
 
-if(BUILD_OFI_TRANSPORT)
-  find_package2(PRIVATE asiofi REQUIRED VERSION 0.5)
-  find_package2(PRIVATE OFI REQUIRED)
-endif()
-
-if(BUILD_PMIX_PLUGIN)
-  find_package2(PRIVATE PMIx REQUIRED VERSION 2.1.4)
-endif()
-
 if(BUILD_FAIRMQ OR BUILD_TIDY_TOOL)
   find_package2(PUBLIC FairLogger REQUIRED VERSION 1.6.0)
   find_package2(PUBLIC Boost REQUIRED VERSION 1.66
@@ -34,14 +25,6 @@ if(BUILD_FAIRMQ OR BUILD_TIDY_TOOL)
   )
 endif()
 
-if(BUILD_OFI_TRANSPORT)
-  set(__old ${CMAKE_FIND_PACKAGE_PREFER_CONFIG})
-  set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ON)
-  find_package2(PUBLIC asio REQUIRED VERSION 1.18)
-  set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ${__old})
-  unset(__old)
-endif()
-
 if(BUILD_FAIRMQ)
   find_package2(PRIVATE ZeroMQ REQUIRED VERSION 4.1.4)
   if(NOT PicoSHA2_BUNDLED)
@@ -87,10 +70,6 @@ if(PROJECT_PACKAGE_DEPENDENCIES)
       if(NOT FairLogger_PREFIX AND FairLogger_ROOT)
         set(FairLogger_PREFIX ${FairLogger_ROOT})
       endif()
-    elseif(${dep} STREQUAL asiofi)
-      if(NOT asiofi_PREFIX AND asiofi_ROOT)
-        set(asiofi_PREFIX ${asiofi_ROOT})
-      endif()
     elseif(${dep} STREQUAL Boost)
       if(TARGET Boost::headers)
         get_target_property(boost_include Boost::headers INTERFACE_INCLUDE_DIRECTORIES)

cmake/FairMQPackage.cmake

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (C) 2018-2021 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
+# Copyright (C) 2018-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
 #                                                                              #
 #              This software is distributed under the terms of the             #
 #              GNU Lesser General Public Licence (LGPL) version 3,             #
@@ -34,8 +34,6 @@ endfunction()
 
 # Configure/Install CMake package
 macro(install_cmake_package)
-  list(SORT PROJECT_PACKAGE_DEPENDENCIES)
-  list(SORT PROJECT_INTERFACE_PACKAGE_DEPENDENCIES)
   include(CMakePackageConfigHelpers)
   set(PACKAGE_INSTALL_DESTINATION
     ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}-${PROJECT_VERSION}

cmake/FairMQProjectSettings.cmake

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (C) 2018-2021 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
+# Copyright (C) 2018-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
 #                                                                              #
 #              This software is distributed under the terms of the             #
 #              GNU Lesser General Public Licence (LGPL) version 3,             #
@@ -114,8 +114,8 @@ endif()
 set(CMAKE_CONFIGURATION_TYPES "Debug" "Release" "RelWithDebInfo")
 set(_warnings "-Wshadow -Wall -Wextra -Wpedantic")
 set(CMAKE_CXX_FLAGS_DEBUG          "-Og -g ${_warnings} ${_sanitizers}")
-set(CMAKE_CXX_FLAGS_RELEASE        "-O2 -DNDEBUG")
-set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g ${_warnings} -DNDEBUG ${_sanitizers}")
+set(CMAKE_CXX_FLAGS_RELEASE        "-O3 -DNDEBUG")
+set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g ${_warnings} -DNDEBUG ${_sanitizers}")
 unset(_warnings)
 unset(_sanitizers)
 
@@ -168,9 +168,22 @@ if(CMAKE_GENERATOR STREQUAL Ninja AND ENABLE_CCACHE)
   endif()
 endif()
 
-if(    CMAKE_CXX_COMPILER_ID STREQUAL "GNU"
-   AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9)
-  set(FAIRMQ_HAS_STD_FILESYSTEM 0)
-else()
-  set(FAIRMQ_HAS_STD_FILESYSTEM 1)
+if(NOT DEFINED FAIRMQ_HAS_STD_FILESYSTEM)
+  if(   (    CMAKE_CXX_COMPILER_ID STREQUAL "GNU"
+         AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9)
+     OR (    CMAKE_CXX_COMPILER_ID STREQUAL "Clang"
+         AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9))
+    set(FAIRMQ_HAS_STD_FILESYSTEM 0)
+  else()
+    set(FAIRMQ_HAS_STD_FILESYSTEM 1)
+  endif()
+endif()
+
+if(NOT DEFINED FAIRMQ_HAS_STD_PMR)
+  if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+    # Clang (to be more precise: libc++) currently does not implement <memory_resource>
+    set(FAIRMQ_HAS_STD_PMR 0)
+  else()
+    set(FAIRMQ_HAS_STD_PMR 1)
+  endif()
 endif()

cmake/FairMQSummary.cmake

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (C) 2018-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
+# Copyright (C) 2018-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
 #                                                                              #
 #              This software is distributed under the terms of the             #
 #              GNU Lesser General Public Licence (LGPL) version 3,             #
@@ -27,18 +27,6 @@ macro(fairmq_summary_components)
     set(tests_summary "${BRed} NO${CR}    (default, enable with ${BMagenta}-DBUILD_TESTING=ON${CR})")
   endif()
   message(STATUS "  ${BWhite}tests${CR}              ${tests_summary}")
-  if(BUILD_OFI_TRANSPORT)
-    set(ofi_summary "${BGreen}YES${CR}    EXPERIMENTAL (disable with ${BMagenta}-DBUILD_OFI_TRANSPORT=OFF${CR})")
-  else()
-    set(ofi_summary "${BRed} NO${CR}    EXPERIMENTAL (default, enable with ${BMagenta}-DBUILD_OFI_TRANSPORT=ON${CR})")
-  endif()
-  message(STATUS "  ${BWhite}ofi_transport${CR}      ${ofi_summary}")
-  if(BUILD_PMIX_PLUGIN)
-    set(pmix_summary "${BGreen}YES${CR}    EXPERIMENTAL (disable with ${BMagenta}-DBUILD_PMIX_PLUGIN=OFF${CR})")
-  else()
-    set(pmix_summary "${BRed} NO${CR}    EXPERIMENTAL (default, enable with ${BMagenta}-DBUILD_PMIX_PLUGIN=ON${CR})")
-  endif()
-  message(STATUS "  ${BWhite}pmix_plugin${CR}        ${pmix_summary}")
   if(BUILD_EXAMPLES)
     set(examples_summary "${BGreen}YES${CR}    (default, disable with ${BMagenta}-DBUILD_EXAMPLES=OFF${CR})")
   else()
@@ -105,3 +93,13 @@ macro(fairmq_summary_debug_mode)
     message(STATUS "  ${Cyan}DEBUG MODE${CR}         ${BRed}${FAIRMQ_DEBUG_MODE}${CR} (enable with ${BMagenta}-DFAIRMQ_DEBUG_MODE=ON${CR})")
   endif()
 endmacro()
+
+macro(fairmq_summary_compile_definitions)
+  message(STATUS "  ")
+  message(STATUS "  ${Cyan}COMPILE DEFINITION               VALUE${CR}")
+  message(STATUS "  ${BWhite}FAIRMQ_HAS_STD_FILESYSTEM${CR}        ${FAIRMQ_HAS_STD_FILESYSTEM}  (overridable with ${BMagenta}-DFAIRMQ_HAS_STD_FILESYSTEM=0|1${CR})")
+  message(STATUS "  ${BWhite}FAIRMQ_HAS_STD_PMR${CR}               ${FAIRMQ_HAS_STD_PMR}  (overridable with ${BMagenta}-DFAIRMQ_HAS_STD_PMR=0|1${CR})")
+  if(DEFINED FAIRMQ_CHANNEL_DEFAULT_AUTOBIND)
+    message(STATUS "  ${BWhite}FAIRMQ_CHANNEL_DEFAULT_AUTOBIND${CR}  ${FAIRMQ_CHANNEL_DEFAULT_AUTOBIND}")
+  endif()
+endmacro()

cmake/FindPMIx.cmake

@@ -1,74 +0,0 @@
-################################################################################
-#    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
-#                                                                              #
-#              This software is distributed under the terms of the             #
-#              GNU Lesser General Public Licence (LGPL) version 3,             #
-#                  copied verbatim in the file "LICENSE"                       #
-################################################################################
-
-
-# The "lib/${CMAKE_LIBRARY_ARCHITECTURE}/pmix" part in all
-# the PATH_SUFFIXES is here to be able to find Debian's
-# libpmix-dev package. It installs everything below
-# /usr/lib/${CMAKE_LIBRARY_ARCHITECTURE}/pmix
-
-
-find_path(PMIx_INCLUDE_DIR
-  NAMES pmix.h
-  HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
-  PATH_SUFFIXES include lib/${CMAKE_LIBRARY_ARCHITECTURE}/pmix/include
-)
-
-find_path(PMIx_LIBRARY_DIR
-  NAMES libpmix.dylib libpmix.so
-  HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
-  PATH_SUFFIXES lib lib64 lib/${CMAKE_LIBRARY_ARCHITECTURE}/pmix/lib
-)
-
-find_library(PMIx_LIBRARY_SHARED
-  NAMES libpmix.dylib libpmix.so
-  HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
-  PATH_SUFFIXES lib lib64 lib/${CMAKE_LIBRARY_ARCHITECTURE}/pmix/lib
-)
-
-find_file(PMIx_VERSION_FILE
-  NAMES pmix_version.h
-  HINTS ${PMIX_ROOT} $ENV{PMIX_ROOT}
-  PATH_SUFFIXES include lib/${CMAKE_LIBRARY_ARCHITECTURE}/pmix/include
-)
-
-file(READ "${PMIx_VERSION_FILE}" __version_raw)
-string(REGEX MATCH "#define PMIX_VERSION_MAJOR ([0-9]?)L?"
-  __version_major_raw "${__version_raw}"
-)
-set(PMIx_VERSION_MAJOR "${CMAKE_MATCH_1}")
-
-string(REGEX MATCH "#define PMIX_VERSION_MINOR ([0-9]?)L?"
-  __version_minor_raw "${__version_raw}"
-)
-set(PMIx_VERSION_MINOR "${CMAKE_MATCH_1}")
-
-string(REGEX MATCH "#define PMIX_VERSION_RELEASE ([0-9]?)L?"
-  __version_patch_raw "${__version_raw}"
-)
-set(PMIx_VERSION_PATCH "${CMAKE_MATCH_1}")
-
-set(PMIx_VERSION "${PMIx_VERSION_MAJOR}.${PMIx_VERSION_MINOR}.${PMIx_VERSION_PATCH}")
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(PMIx
-  REQUIRED_VARS
-  PMIx_INCLUDE_DIR
-  PMIx_LIBRARY_DIR
-  PMIx_LIBRARY_SHARED
-
-  VERSION_VAR PMIx_VERSION
-)
-
-if(NOT TARGET PMIx::libpmix AND PMIx_FOUND)
-  add_library(PMIx::libpmix SHARED IMPORTED)
-  set_target_properties(PMIx::libpmix PROPERTIES
-    IMPORTED_LOCATION ${PMIx_LIBRARY_SHARED}
-    INTERFACE_INCLUDE_DIRECTORIES ${PMIx_INCLUDE_DIR}
-  )
-endif()

cmake/GTestHelper.cmake

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (C) 2017-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
+# Copyright (C) 2017-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
 #                                                                              #
 #              This software is distributed under the terms of the             #
 #              GNU Lesser General Public Licence (LGPL) version 3,             #
@@ -20,6 +20,7 @@
 #                        [DEPENDS dep1 [dep2 ...]]
 #                        [LINKS linklib1 [linklib2 ...]
 #                        [INCLUDES dir1 [dir2 ...]
+#                        [ENVIRONMENT var1=op:val1[;var2=op:val2 ...]]
 #                        [TIMEOUT seconds]
 #                        [RUN_SERIAL ON/OFF])
 #
@@ -56,7 +57,7 @@ function(add_testsuite suitename)
     cmake_parse_arguments(testsuite
         ""
         "TIMEOUT;RUN_SERIAL"
-        "SOURCES;LINKS;DEPENDS;INCLUDES;DEFINITIONS"
+        "SOURCES;LINKS;DEPENDS;INCLUDES;DEFINITIONS;ENVIRONMENT"
         ${ARGN}
     )
 
@@ -77,6 +78,9 @@ function(add_testsuite suitename)
     if(testsuite_DEFINITIONS)
         target_compile_definitions("${target}" PUBLIC ${testsuite_DEFINITIONS})
     endif()
+    if(testsuite_ENVIRONMENT AND CMAKE_VERSION VERSION_GREATER_EQUAL 3.22)
+        set(env "ENVIRONMENT_MODIFICATION" ${testsuite_ENVIRONMENT})
+    endif()
     if(BUILD_TIDY_TOOL AND RUN_FAIRMQ_TIDY)
       fairmq_target_tidy(TARGET ${target})
     endif()
@@ -97,6 +101,7 @@ function(add_testsuite suitename)
       TEST_PREFIX ${suitename}.
       PROPERTIES RUN_SERIAL ${testsuite_RUN_SERIAL}
                  TIMEOUT ${testsuite_TIMEOUT}
+                 ${env}
     )
 
     list(APPEND ALL_TEST_TARGETS ${target})

codemeta.json

@@ -6,17 +6,29 @@
   "license": "./COPYRIGHT",
   "datePublished": "2018-04-15",
   "developmentStatus": "active",
+  "softwareVersion": "master",
+  "releaseNotes": "https://github.com/FairRootGroup/FairMQ/releases",
   "codeRepository": "https://github.com/FairRootGroup/FairMQ/",
+  "readme": "https://github.com/FairRootGroup/FairMQ/#readme",
   "issueTracker": "https://github.com/FairRootGroup/FairMQ/issues",
   "identifier": "https://doi.org/10.5281/zenodo.1689985",
+  "maintainer": [
+    {
+      "@type": "ResearchOrganisation",
+      "@id": "https://ror.org/02k8cbn47",
+      "name": "GSI Helmholtz Centre for Heavy Ion Research"
+    }
+  ],
   "author": [
     {
       "@type": "Person",
+      "@id": "https://orcid.org/0000-0002-8071-4497",
       "givenName": "Mohammad",
       "familyName": "Al-Turany"
     },
     {
       "@type": "Person",
+      "@id": "https://orcid.org/0000-0003-3787-1910",
       "givenName": "Dennis",
       "familyName": "Klein"
     },
@@ -27,6 +39,7 @@
     },
     {
       "@type": "Person",
+      "@id": "https://orcid.org/0000-0002-6249-155X",
       "givenName": "Alexey",
       "familyName": "Rybalchenko"
     },
@@ -91,6 +104,7 @@
     },
     {
       "@type": "Person",
+      "@id": "https://orcid.org/0000-0002-5321-8404",
       "givenName": "Christian",
       "familyName": "Tacke"
     },

codemeta_update.py

@@ -1,83 +0,0 @@
-#! /usr/bin/env python3
-# Copyright (C) 2021 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH
-#
-# SPDX-License-Identifier: LGPL-3.0-or-later
-
-
-import argparse
-import json
-import re
-from collections import OrderedDict
-
-
-class CodeMetaManipulator(object):
-    def load(self, filename='codemeta.json'):
-        with open(filename, 'rb') as fp:
-            self.data = json.load(fp, object_pairs_hook=OrderedDict)
-
-    @staticmethod
-    def _dict_entry_cmp(dict1, dict2, field):
-        if (field in dict1) and (field in dict2):
-            return dict1[field] == dict2[field]
-        else:
-            return False
-
-    @classmethod
-    def find_person_entry(cls, person_list, matchdict):
-        for entry in person_list:
-            if cls._dict_entry_cmp(entry, matchdict, 'email'):
-                return entry
-            if cls._dict_entry_cmp(entry, matchdict, 'familyName') \
-                    and cls._dict_entry_cmp(entry, matchdict, 'givenName'):
-                return entry
-        return None
-
-    @staticmethod
-    def update_person_entry(entry, matchdict):
-        if entry is None:
-            entry = OrderedDict()
-            entry['@type'] = 'Person'
-        for field in ('givenName', 'familyName', 'email'):
-            val = matchdict.get(field, None)
-            if val is not None:
-                entry[field] = val
-        return entry
-
-    def handle_person_list_file(self, filename, cm_field_name):
-        fp = open(filename, 'r', encoding='utf8')
-        findregex = re.compile(r'^(?P<familyName>[-\w\s]*[-\w]),\s*'
-                               r'(?P<givenName>[-\w\s]*[-\w])\s*'
-                               r'(?:<(?P<email>\S+@\S+)>)?$')
-        person_list = self.data.setdefault(cm_field_name, [])
-        for line in fp:
-            line = line.strip()
-            m = findregex.match(line)
-            if m is None:
-                raise RuntimeError("Could not analyze line %r" % line)
-            found_entry = self.find_person_entry(person_list, m.groupdict())
-            entry = self.update_person_entry(found_entry, m.groupdict())
-            if found_entry is None:
-                person_list.append(entry)
-
-    def save(self, filename='codemeta.json'):
-        with open('codemeta.json', 'w', encoding='utf8') as fp:
-            json.dump(self.data, fp, indent=2)
-            fp.write('\n')
-
-
-def main():
-    parser = argparse.ArgumentParser(description='Update codemeta.json')
-    parser.add_argument('--set-version', dest='newversion')
-    args = parser.parse_args()
-
-    cm = CodeMetaManipulator()
-    cm.load()
-    if args.newversion is not None:
-        cm.data['softwareVersion'] = args.newversion
-    cm.handle_person_list_file('AUTHORS', 'author')
-    cm.handle_person_list_file('CONTRIBUTORS', 'contributor')
-    cm.save()
-
-
-if __name__ == '__main__':
-    main()

docs/Configuration.md

@@ -28,7 +28,6 @@ Here is an overview of the device/channel options and when they are applied:
 | `init-timeout` | at the end of `fair::mq::State::InitializingDevice` |
 | `shm-segment-size` | at the end of `fair::mq::State::InitializingDevice` |
 | `shm-monitor` | at the end of `fair::mq::State::InitializingDevice` |
-| `ofi-size-hint` | at the end of `fair::mq::State::InitializingDevice` |
 | `rate` | at the end of `fair::mq::State::InitializingDevice` |
 | `session` | at the end of `fair::mq::State::InitializingDevice` |
 | `chan.*` | at the end of `fair::mq::State::InitializingDevice` (channel addresses can be also applied during `fair::mq::State::Binding`/`fair::mq::State::Connecting`) |

docs/Plugins.md

@@ -56,8 +56,6 @@ A more complete example which may serve as a start including example CMake code
 
 ### 7.3.1 PMIx
 
-The [PMIx](https://pmix.org/) plugin enables launching a FairMQ topology with any PMIx capable launcher, e.g. the [Open Run-Time Environment (ORTE) of OpenMPI](https://www.open-mpi.org/doc/v4.0/man1/mpirun.1.php) or the [Slurm workload manager](https://slurm.schedmd.com/srun.html). This plugin is not (yet) very mature and serves as a proof of concept at the moment.
-
-TODO example usage
+The [PMIx](https://pmix.org/) plugin enables launching a FairMQ topology with any PMIx capable launcher, e.g. the [Open Run-Time Environment (ORTE) of OpenMPI](https://www.open-mpi.org/doc/v4.0/man1/mpirun.1.php) or the [Slurm workload manager](https://slurm.schedmd.com/srun.html). This experimental plugin has been last released in v1.4.56 and is removed in v1.5+. For now there are no plans to pick up development of it again.
 
 ← [Back](../README.md)

examples/1-1/CMakeLists.txt

@@ -1,51 +1,9 @@
 ################################################################################
-#    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+# Copyright (C) 2014-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
 #                                                                              #
 #              This software is distributed under the terms of the             #
 #              GNU Lesser General Public Licence (LGPL) version 3,             #
 #                  copied verbatim in the file "LICENSE"                       #
 ################################################################################
 
-add_executable(fairmq-ex-1-1-sampler sampler.cxx)
-target_link_libraries(fairmq-ex-1-1-sampler PRIVATE FairMQ)
-
-add_executable(fairmq-ex-1-1-sink sink.cxx)
-target_link_libraries(fairmq-ex-1-1-sink PRIVATE FairMQ)
-
-add_custom_target(Example11 DEPENDS fairmq-ex-1-1-sampler fairmq-ex-1-1-sink)
-
-set(EX_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_BINARY_DIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-1-1.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-1-1.sh)
-
-# test
-
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-1-1.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh)
-
-add_test(NAME Example.1-1.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh zeromq)
-set_tests_properties(Example.1-1.zeromq PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Received: ")
-
-add_test(NAME Example.1-1.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-1.sh shmem)
-set_tests_properties(Example.1-1.shmem PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Received: ")
-
-# install
-
-install(
-  TARGETS
-  fairmq-ex-1-1-sampler
-  fairmq-ex-1-1-sink
-
-  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
-  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
-)
-
-# configure run script with different executable paths for build and for install directories
-set(EX_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-1-1.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-1-1.sh_install)
-
-install(
-  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-1-1.sh_install
-  DESTINATION ${PROJECT_INSTALL_BINDIR}
-  RENAME fairmq-start-ex-1-1.sh
-)
+add_example(NAME 1-1 DEVICE sampler sink)

examples/1-1/test-ex-1-1.sh.in

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
 
 transport="zeromq"
@@ -13,7 +15,7 @@ chan="data"
 chanAddr="/tmp/fmq_$session""_""$chan""_""$transport"
 
 # setup a trap to kill everything if the test fails/timeouts
-trap 'kill -TERM $SAMPLER_PID; kill -TERM $SINK_PID; wait $SAMPLER_PID; wait $SINK_PID; rm $chanAddr' TERM
+trap 'set +e; kill -TERM $SAMPLER_PID; kill -TERM $SINK_PID; wait $SAMPLER_PID; wait $SINK_PID; rm $chanAddr; exit 0' TERM
 
 SAMPLER="fairmq-ex-1-1-sampler"
 SAMPLER+=" --id sampler1"
@@ -22,6 +24,7 @@ SAMPLER+=" --transport $transport"
 SAMPLER+=" --verbosity veryhigh"
 SAMPLER+=" --session $session"
 SAMPLER+=" --shm-segment-size 100000000"
+SAMPLER+=" --shm-monitor true"
 SAMPLER+=" --control static --color false"
 SAMPLER+=" --max-iterations 1"
 SAMPLER+=" --channel-config name=$chan,type=push,method=bind,address=ipc://$chanAddr,rateLogging=0"
@@ -34,6 +37,7 @@ SINK+=" --transport $transport"
 SINK+=" --verbosity veryhigh"
 SINK+=" --session $session"
 SINK+=" --shm-segment-size 100000000"
+SINK+=" --shm-monitor true"
 SINK+=" --control static --color false"
 SINK+=" --max-iterations 1"
 SINK+=" --channel-config name=$chan,type=pull,method=connect,address=ipc://$chanAddr,rateLogging=0"
@@ -44,4 +48,6 @@ SINK_PID=$!
 wait $SAMPLER_PID
 wait $SINK_PID
 
+set +e
 rm $chanAddr
+exit 0

examples/1-n-1/CMakeLists.txt

@@ -1,64 +1,9 @@
 ################################################################################
-#    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+# Copyright (C) 2014-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
 #                                                                              #
 #              This software is distributed under the terms of the             #
 #              GNU Lesser General Public Licence (LGPL) version 3,             #
 #                  copied verbatim in the file "LICENSE"                       #
 ################################################################################
 
-add_executable(fairmq-ex-1-n-1-sampler sampler.cxx)
-target_link_libraries(fairmq-ex-1-n-1-sampler PRIVATE FairMQ)
-
-add_executable(fairmq-ex-1-n-1-processor processor.cxx)
-target_link_libraries(fairmq-ex-1-n-1-processor PRIVATE FairMQ)
-
-add_executable(fairmq-ex-1-n-1-sink sink.cxx)
-target_link_libraries(fairmq-ex-1-n-1-sink PRIVATE FairMQ)
-
-add_custom_target(Example1N1 DEPENDS fairmq-ex-1-n-1-sampler fairmq-ex-1-n-1-processor fairmq-ex-1-n-1-sink)
-
-set(EX_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(EX_CONF_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_BINARY_DIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-1-n-1.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-1-n-1.sh)
-
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ex-1-n-1.json ${CMAKE_CURRENT_BINARY_DIR}/ex-1-n-1.json)
-
-# test
-
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-1-n-1.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh)
-
-add_test(NAME Example.1-n-1.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh zeromq)
-set_tests_properties(Example.1-n-1.zeromq PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Received: ")
-
-add_test(NAME Example.1-n-1.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-1-n-1.sh shmem)
-set_tests_properties(Example.1-n-1.shmem PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Received: ")
-
-# install
-
-install(
-  TARGETS
-  fairmq-ex-1-n-1-sampler
-  fairmq-ex-1-n-1-processor
-  fairmq-ex-1-n-1-sink
-
-  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
-  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
-)
-
-# configure run script with different executable paths for build and for install directories
-set(EX_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
-set(EX_CONF_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_DATADIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-1-n-1.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-1-n-1.sh_install)
-
-install(
-  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-1-n-1.sh_install
-  DESTINATION ${PROJECT_INSTALL_BINDIR}
-  RENAME fairmq-start-ex-1-n-1.sh
-)
-
-install(
-  FILES ${CMAKE_CURRENT_BINARY_DIR}/ex-1-n-1.json
-  DESTINATION ${PROJECT_INSTALL_DATADIR}
-)
+add_example(NAME 1-n-1 DEVICE sampler processor sink CONFIG)

examples/1-n-1/test-ex-1-n-1.sh.in

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
 
 transport="zeromq"
@@ -15,7 +17,7 @@ chan1Addr="/tmp/fmq_$session""_""$chan1""_""$transport"
 chan2Addr="/tmp/fmq_$session""_""$chan2""_""$transport"
 
 # setup a trap to kill everything if the test fails/timeouts
-trap 'kill -TERM $SAMPLER_PID; kill -TERM $SINK_PID; kill -TERM $PROCESSOR1_PID; kill -TERM $PROCESSOR2_PID; wait $SAMPLER_PID; wait $SINK_PID; wait $PROCESSOR1_PID; wait $PROCESSOR2_PID; rm $chan1Addr; rm $chan2Addr' TERM
+trap 'set +e; kill -TERM $SAMPLER_PID; kill -TERM $SINK_PID; kill -TERM $PROCESSOR1_PID; kill -TERM $PROCESSOR2_PID; wait $SAMPLER_PID; wait $SINK_PID; wait $PROCESSOR1_PID; wait $PROCESSOR2_PID; rm $chan1Addr; rm $chan2Addr; exit 0' TERM
 
 SAMPLER="fairmq-ex-1-n-1-sampler"
 SAMPLER+=" --id sampler1"
@@ -24,6 +26,7 @@ SAMPLER+=" --verbosity veryhigh"
 SAMPLER+=" --session $session"
 SAMPLER+=" --severity debug"
 SAMPLER+=" --shm-segment-size 100000000"
+SAMPLER+=" --shm-monitor true"
 SAMPLER+=" --control static --color false"
 SAMPLER+=" --max-iterations 2"
 SAMPLER+=" --channel-config name=$chan1,type=push,method=bind,address=ipc://$chan1Addr,rateLogging=0"
@@ -37,6 +40,7 @@ PROCESSOR1+=" --verbosity veryhigh"
 PROCESSOR1+=" --session $session"
 PROCESSOR1+=" --severity debug"
 PROCESSOR1+=" --shm-segment-size 100000000"
+PROCESSOR1+=" --shm-monitor true"
 PROCESSOR1+=" --control static --color false"
 PROCESSOR1+=" --channel-config name=$chan1,type=pull,method=connect,address=ipc://$chan1Addr,rateLogging=0"
 PROCESSOR1+="                  name=$chan2,type=push,method=connect,address=ipc://$chan2Addr,rateLogging=0"
@@ -50,6 +54,7 @@ PROCESSOR2+=" --verbosity veryhigh"
 PROCESSOR2+=" --session $session"
 PROCESSOR2+=" --severity debug"
 PROCESSOR2+=" --shm-segment-size 100000000"
+PROCESSOR2+=" --shm-monitor true"
 PROCESSOR2+=" --control static --color false"
 PROCESSOR2+=" --channel-config name=$chan1,type=pull,method=connect,address=ipc://$chan1Addr,rateLogging=0"
 PROCESSOR2+="                  name=$chan2,type=push,method=connect,address=ipc://$chan2Addr,rateLogging=0"
@@ -63,6 +68,7 @@ SINK+=" --verbosity veryhigh"
 SINK+=" --session $session"
 SINK+=" --severity debug"
 SINK+=" --shm-segment-size 100000000"
+SINK+=" --shm-monitor true"
 SINK+=" --control static --color false"
 SINK+=" --max-iterations 2"
 SINK+=" --channel-config name=$chan2,type=pull,method=bind,address=ipc://$chan2Addr,rateLogging=0"
@@ -81,4 +87,6 @@ kill -SIGINT $PROCESSOR2_PID
 wait $PROCESSOR1_PID
 wait $PROCESSOR2_PID
 
+set +e
 rm $chan1Addr; rm $chan2Addr
+exit 0

examples/CMakeLists.txt

@@ -1,15 +1,145 @@
 ################################################################################
-#    Copyright (C) 2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+# Copyright (C) 2018-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
 #                                                                              #
 #              This software is distributed under the terms of the             #
 #              GNU Lesser General Public Licence (LGPL) version 3,             #
 #                  copied verbatim in the file "LICENSE"                       #
 ################################################################################
 
+set(exe_prefix "fairmq-ex")
+set(script_prefix "fairmq-start-ex")
+set(test_script_prefix "test-ex")
+set(testsuite "Example")
+set(transports "zeromq" "shmem")
+
+function(add_example)
+  cmake_parse_arguments(PARSE_ARGV 0 ARG
+    "CONFIG;NO_TRANSPORT;NO_TEST"
+    "NAME"
+    "DEVICE;VARIANT;TRANSPORT;SCRIPT"
+  )
+
+  if(ARG_UNPARSED_ARGUMENTS)
+    message(FATAL_ERROR "Unexpected unparsed arguments: ${ARG_UNPARSED_ARGUMENTS}")
+  endif()
+
+  if(ARG_NAME)
+    set(name ${ARG_NAME})
+  else()
+    message(FATAL_ERROR "NAME arg is required")
+  endif()
+
+  if(ENABLE_SANITIZER_LEAK AND CMAKE_VERSION VERSION_GREATER_EQUAL 3.22)
+    get_filename_component(lsan_supps "${CMAKE_SOURCE_DIR}/test/leak_sanitizer_suppressions.txt" ABSOLUTE)
+    set(lsan_options "LSAN_OPTIONS=set:suppressions=${lsan_supps}")
+  endif()
+
+  if(ARG_DEVICE)
+    set(exe_targets)
+    foreach(device IN LISTS ARG_DEVICE)
+      set(exe "${exe_prefix}-${name}-${device}")
+      list(APPEND exe_targets ${exe})
+      add_executable(${exe} "${device}.cxx")
+      target_link_libraries(${exe} PRIVATE FairMQ)
+    endforeach()
+  endif()
+
+  if(ARG_TRANSPORT)
+    set(transports ${ARG_TRANSPORT})
+  endif()
+
+  if(ARG_SCRIPT)
+    set(scripts ${ARG_SCRIPT})
+  else()
+    set(scripts ${ARG_NAME})
+  endif()
+
+  set(EX_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR})
+  if(ARG_CONFIG)
+    set(EX_CONF_DIR ${CMAKE_CURRENT_BINARY_DIR})
+  endif()
+  set(FAIRMQ_BIN_DIR ${CMAKE_BINARY_DIR}/fairmq)
+  foreach(script IN LISTS scripts)
+    set(script_file "${script_prefix}-${script}.sh")
+    configure_file("${CMAKE_CURRENT_SOURCE_DIR}/${script_file}.in" "${CMAKE_CURRENT_BINARY_DIR}/${script_file}" @ONLY)
+  endforeach()
+
+  if(ARG_CONFIG)
+    set(config "ex-${name}.json")
+    configure_file("${CMAKE_CURRENT_SOURCE_DIR}/${config}" "${CMAKE_CURRENT_BINARY_DIR}/${config}")
+  endif()
+
+  # test
+  if(NOT ARG_NO_TEST)
+    set(test_script "${test_script_prefix}-${name}.sh")
+    configure_file("${CMAKE_CURRENT_SOURCE_DIR}/${test_script}.in" "${CMAKE_CURRENT_BINARY_DIR}/${test_script}")
+
+    if(ARG_NO_TRANSPORT)
+      set(test "${testsuite}.${name}.${transport}")
+      add_test(NAME ${test} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${test_script} ${transport})
+      set_tests_properties(${test} PROPERTIES TIMEOUT "30")
+      if(lsan_options)
+        set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION ${lsan_options})
+      endif()
+    else()
+      foreach(transport IN LISTS transports)
+        if(ARG_VARIANT)
+          foreach(variant IN LISTS ARG_VARIANT)
+            set(test "${testsuite}.${name}.${variant}.${transport}")
+            add_test(NAME ${test} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${test_script} ${transport} ${variant})
+            set_tests_properties(${test} PROPERTIES TIMEOUT "30")
+            if(lsan_options)
+              set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION ${lsan_options})
+            endif()
+          endforeach()
+        else()
+          set(test "${testsuite}.${name}.${transport}")
+          add_test(NAME ${test} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${test_script} ${transport})
+          set_tests_properties(${test} PROPERTIES TIMEOUT "30")
+          if(lsan_options)
+            set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION ${lsan_options})
+          endif()
+        endif()
+      endforeach()
+    endif()
+  endif()
+
+  # install
+  install(
+    TARGETS ${exe_targets}
+    LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
+    RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
+  )
+
+  # configure run script with different executable paths for build and for install directories
+  set(EX_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
+  if(ARG_CONFIG)
+    set(EX_CONF_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_DATADIR})
+  endif()
+  set(FAIRMQ_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR}/fairmq)
+  foreach(script IN LISTS scripts)
+    set(script_file "${script_prefix}-${script}.sh")
+    configure_file("${CMAKE_CURRENT_SOURCE_DIR}/${script_file}.in" "${CMAKE_CURRENT_BINARY_DIR}/${script_file}_install" @ONLY)
+    install(
+      PROGRAMS "${CMAKE_CURRENT_BINARY_DIR}/${script_file}_install"
+      DESTINATION ${PROJECT_INSTALL_BINDIR}
+      RENAME ${script_file}
+    )
+  endforeach()
+
+  if(ARG_CONFIG)
+    install(
+      FILES ${CMAKE_CURRENT_BINARY_DIR}/${config}
+      DESTINATION ${PROJECT_INSTALL_DATADIR}
+    )
+  endif()
+endfunction()
+
 add_subdirectory(1-1)
 add_subdirectory(1-n-1)
 add_subdirectory(builtin-devices)
 add_subdirectory(copypush)
+add_subdirectory(custom-controller)
 add_subdirectory(dds)
 add_subdirectory(multipart)
 add_subdirectory(multiple-channels)

examples/builtin-devices/CMakeLists.txt

@@ -1,40 +1,11 @@
- ################################################################################
- #    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
- #                                                                              #
- #              This software is distributed under the terms of the             #
- #              GNU Lesser General Public Licence (LGPL) version 3,             #
- #                  copied verbatim in the file "LICENSE"                       #
- ################################################################################
+################################################################################
+# Copyright (C) 2014-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
 
-set(EX_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_BINARY_DIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-builtin-devices.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-builtin-devices.sh)
-
-# test
-
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-builtin-devices.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh)
-
-add_test(NAME Example.BuiltinDevices.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq)
-set_tests_properties(Example.BuiltinDevices.zeromq PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
-
-add_test(NAME Example.BuiltinDevices.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem)
-set_tests_properties(Example.BuiltinDevices.shmem PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
-
-add_test(NAME Example.BuiltinDevices.multipart.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh zeromq true 2)
-set_tests_properties(Example.BuiltinDevices.multipart.zeromq PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
-
-add_test(NAME Example.BuiltinDevices.multipart.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-builtin-devices.sh shmem true 2)
-set_tests_properties(Example.BuiltinDevices.multipart.shmem PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Configured maximum number of iterations reached")
-
-# install
-
-# configure run script with different executable paths for build and for install directories
-set(EX_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-builtin-devices.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-builtin-devices.sh_install)
-
-install(
-  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-builtin-devices.sh_install
-  DESTINATION ${PROJECT_INSTALL_BINDIR}
-  RENAME fairmq-start-ex-builtin-devices.sh
+add_example(NAME builtin-devices
+  VARIANT single_msg multipart
 )

examples/builtin-devices/test-ex-builtin-devices.sh.in

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
 
 transport="zeromq"
@@ -10,12 +12,9 @@ if [[ $1 =~ ^[a-z]+$ ]]; then
     transport=$1
 fi
 
-if [[ $2 =~ ^[a-z]+$ ]]; then
-    multipart=$2
-fi
-
-if [[ $3 =~ ^[0-9]+$ ]]; then
-    numParts=$3
+if [[ $2 =~ ^multipart$ ]]; then
+    multipart="true"
+    numParts=2
 fi
 
 session="$(@CMAKE_BINARY_DIR@/fairmq/fairmq-uuid-gen -h)"
@@ -33,7 +32,7 @@ chan4Addr="/tmp/fmq_$session""_""$chan4""_""$transport"
 chan5Addr="/tmp/fmq_$session""_""$chan5""_""$transport"
 
 # setup a trap to kill everything if the test fails/timeouts
-trap 'kill -TERM $SAMPLER_PID; kill -TERM $SPLITTER_PID; kill -TERM $PROXY1_PID; kill -TERM $PROXY2_PID; kill -TERM $MERGER_PID; kill -TERM $MULTIPLIER_PID; kill -TERM $SINK_PID; rm $chan1Addr; rm $chan2Addr1; rm $chan2Addr2; rm $chan3Addr1; rm $chan3Addr2; rm $chan4Addr; rm $chan5Addr' TERM
+trap 'set +e; kill -TERM $SAMPLER_PID; kill -TERM $SPLITTER_PID; kill -TERM $PROXY1_PID; kill -TERM $PROXY2_PID; kill -TERM $MERGER_PID; kill -TERM $MULTIPLIER_PID; kill -TERM $SINK_PID; rm $chan1Addr; rm $chan2Addr1; rm $chan2Addr2; rm $chan3Addr1; rm $chan3Addr2; rm $chan4Addr; rm $chan5Addr; exit 0' TERM
 
 SAMPLER="fairmq-bsampler"
 SAMPLER+=" --id bsampler1"
@@ -43,6 +42,7 @@ SAMPLER+=" --color false"
 SAMPLER+=" --control static"
 SAMPLER+=" --verbosity veryhigh"
 SAMPLER+=" --shm-segment-size 100000000"
+SAMPLER+=" --shm-monitor true"
 SAMPLER+=" --severity debug"
 SAMPLER+=" --msg-size 100000"
 SAMPLER+=" --multipart $multipart"
@@ -62,6 +62,7 @@ SPLITTER+=" --color false"
 SPLITTER+=" --control static"
 SPLITTER+=" --verbosity veryhigh"
 SPLITTER+=" --shm-segment-size 100000000"
+SPLITTER+=" --shm-monitor true"
 SPLITTER+=" --multipart $multipart"
 SPLITTER+=" --in-channel $chan1"
 SPLITTER+=" --out-channel $chan2"
@@ -78,6 +79,7 @@ PROXY1+=" --color false"
 PROXY1+=" --control static"
 PROXY1+=" --verbosity veryhigh"
 PROXY1+=" --shm-segment-size 100000000"
+PROXY1+=" --shm-monitor true"
 PROXY1+=" --multipart $multipart"
 PROXY1+=" --in-channel $chan2"
 PROXY1+=" --out-channel $chan3"
@@ -94,6 +96,7 @@ PROXY2+=" --color false"
 PROXY2+=" --control static"
 PROXY2+=" --verbosity veryhigh"
 PROXY2+=" --shm-segment-size 100000000"
+PROXY2+=" --shm-monitor true"
 PROXY2+=" --multipart $multipart"
 PROXY2+=" --in-channel $chan2"
 PROXY2+=" --out-channel $chan3"
@@ -110,6 +113,7 @@ MERGER+=" --color false"
 MERGER+=" --control static"
 MERGER+=" --verbosity veryhigh"
 MERGER+=" --shm-segment-size 100000000"
+MERGER+=" --shm-monitor true"
 MERGER+=" --multipart $multipart"
 MERGER+=" --in-channel $chan3"
 MERGER+=" --out-channel $chan4"
@@ -126,6 +130,7 @@ MULTIPLIER+=" --color false"
 MULTIPLIER+=" --control static"
 MULTIPLIER+=" --verbosity veryhigh"
 MULTIPLIER+=" --shm-segment-size 100000000"
+MULTIPLIER+=" --shm-monitor true"
 MULTIPLIER+=" --multipart $multipart"
 MULTIPLIER+=" --in-channel $chan4"
 MULTIPLIER+=" --out-channel $chan5"
@@ -143,6 +148,7 @@ SINK+=" --control static"
 SINK+=" --verbosity veryhigh"
 SINK+=" --severity debug"
 SINK+=" --multipart $multipart"
+SINK+=" --shm-monitor true"
 SINK+=" --max-iterations 2"
 SINK+=" --in-channel $chan5"
 SINK+=" --channel-config name=$chan5,type=pull,method=bind,sndBufSize=50,rcvBufSize=50,address=ipc://$chan5Addr"
@@ -165,4 +171,6 @@ wait $PROXY2_PID
 wait $MERGER_PID
 wait $MULTIPLIER_PID
 
+set +e
 rm $chan1Addr; rm $chan2Addr1; rm $chan2Addr2; rm $chan3Addr1; rm $chan3Addr2; rm $chan4Addr; rm $chan5Addr
+exit 0

examples/copypush/CMakeLists.txt

@@ -1,52 +1,11 @@
 ################################################################################
-#    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
+# Copyright (C) 2014-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
 #                                                                              #
 #              This software is distributed under the terms of the             #
 #              GNU Lesser General Public Licence (LGPL) version 3,             #
 #                  copied verbatim in the file "LICENSE"                       #
 ################################################################################
 
-
-add_executable(fairmq-ex-copypush-sampler sampler.cxx)
-target_link_libraries(fairmq-ex-copypush-sampler PRIVATE FairMQ)
-
-add_executable(fairmq-ex-copypush-sink sink.cxx)
-target_link_libraries(fairmq-ex-copypush-sink PRIVATE FairMQ)
-
-add_custom_target(ExampleCopyPush DEPENDS fairmq-ex-copypush-sampler fairmq-ex-copypush-sink)
-
-set(EX_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_BINARY_DIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-copypush.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-copypush.sh)
-
-# test
-
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-copypush.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh)
-
-add_test(NAME Example.CopyPush.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh zeromq)
-set_tests_properties(Example.CopyPush.zeromq PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Received message: ")
-
-add_test(NAME Example.CopyPush.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-copypush.sh shmem)
-set_tests_properties(Example.CopyPush.shmem PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Received message: ")
-
-# install
-
-install(
-  TARGETS
-  fairmq-ex-copypush-sampler
-  fairmq-ex-copypush-sink
-
-  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
-  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
-)
-
-# configure run script with different executable paths for build and for install directories
-set(EX_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-copypush.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-copypush.sh_install)
-
-install(
-  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-copypush.sh_install
-  DESTINATION ${PROJECT_INSTALL_BINDIR}
-  RENAME fairmq-start-ex-copypush.sh
+add_example(NAME copypush
+  DEVICE sampler sink
 )

examples/copypush/test-ex-copypush.sh.in

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
 
 transport="zeromq"
@@ -14,7 +16,7 @@ chanAddr1="/tmp/fmq_$session""_""$chan""_1""_""$transport"
 chanAddr2="/tmp/fmq_$session""_""$chan""_2""_""$transport"
 
 # setup a trap to kill everything if the test fails/timeouts
-trap 'kill -TERM $SAMPLER_PID; kill -TERM $SINK1_PID; kill -TERM $SINK2_PID; wait $SAMPLER_PID; wait $SINK1_PID; wait $SINK2_PID; rm $chanAddr1; rm $chanAddr2' TERM
+trap 'set +e; kill -TERM $SAMPLER_PID; kill -TERM $SINK1_PID; kill -TERM $SINK2_PID; wait $SAMPLER_PID; wait $SINK1_PID; wait $SINK2_PID; rm $chanAddr1; rm $chanAddr2; exit 0' TERM
 
 SAMPLER="fairmq-ex-copypush-sampler"
 SAMPLER+=" --id sampler1"
@@ -22,6 +24,7 @@ SAMPLER+=" --transport $transport"
 SAMPLER+=" --verbosity veryhigh"
 SAMPLER+=" --severity debug"
 SAMPLER+=" --shm-segment-size 100000000"
+SAMPLER+=" --shm-monitor true"
 SAMPLER+=" --session $session"
 SAMPLER+=" --control static --color false"
 SAMPLER+=" --max-iterations 1"
@@ -35,6 +38,7 @@ SINK1+=" --transport $transport"
 SINK1+=" --verbosity veryhigh"
 SINK1+=" --severity debug"
 SINK1+=" --shm-segment-size 100000000"
+SINK1+=" --shm-monitor true"
 SINK1+=" --session $session"
 SINK1+=" --control static --color false"
 SINK1+=" --max-iterations 1"
@@ -48,6 +52,7 @@ SINK2+=" --transport $transport"
 SINK2+=" --verbosity veryhigh"
 SINK2+=" --severity debug"
 SINK2+=" --shm-segment-size 100000000"
+SINK2+=" --shm-monitor true"
 SINK2+=" --session $session"
 SINK2+=" --control static --color false"
 SINK2+=" --max-iterations 1"
@@ -60,4 +65,6 @@ wait $SAMPLER_PID
 wait $SINK1_PID
 wait $SINK2_PID
 
+set +e
 rm $chanAddr1; rm $chanAddr2
+exit 0

examples/custom-controller/CMakeLists.txt

@@ -0,0 +1,20 @@
+################################################################################
+# Copyright (C) 2022-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
+
+set(name "custom-controller")
+set(exe "${exe_prefix}-${name}")
+add_executable(${exe} main.cxx)
+target_link_libraries(${exe} PRIVATE FairMQ)
+set_target_properties(${exe} PROPERTIES ENABLE_EXPORTS ON)
+
+set(test "${testsuite}.${name}")
+add_test(NAME ${test} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${exe})
+set_tests_properties(${test} PROPERTIES TIMEOUT 30)
+if(lsan_options)
+  set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION ${lsan_options})
+endif()

examples/custom-controller/MyController.h

@@ -0,0 +1,97 @@
+/********************************************************************************
+ *    Copyright (C) 2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_EXAMPLE_CUSTOM_CONTROLLER_PLUGIN
+#define FAIR_MQ_EXAMPLE_CUSTOM_CONTROLLER_PLUGIN
+
+#include <fairmq/Plugin.h>
+#include <utility>   // for std::forward
+
+namespace example {
+
+struct MyController : fair::mq::Plugin   // NOLINT
+{
+    template<typename... Args>
+    MyController(Args&&... args)
+        : Plugin(std::forward<Args>(args)...)
+    {
+        TakeDeviceControl();
+
+        SubscribeToDeviceStateChange([this](auto state) {
+            auto shutdown = GetProperty<bool>("please-shut-me-down", false);
+            try {
+                switch (state) {
+                    case DeviceState::Idle: {
+                        ChangeDeviceState(shutdown ? DeviceStateTransition::End
+                                                    : DeviceStateTransition::InitDevice);
+                        break;
+                    }
+                    case DeviceState::InitializingDevice: {
+                        ChangeDeviceState(DeviceStateTransition::CompleteInit);
+                        break;
+                    }
+                    case DeviceState::Initialized: {
+                        ChangeDeviceState(DeviceStateTransition::Bind);
+                        break;
+                    }
+                    case DeviceState::Bound: {
+                        ChangeDeviceState(DeviceStateTransition::Connect);
+                        break;
+                    }
+                    case DeviceState::DeviceReady: {
+                        ChangeDeviceState(shutdown ? DeviceStateTransition::ResetDevice
+                                                    : DeviceStateTransition::InitTask);
+                        break;
+                    }
+                    case DeviceState::Ready: {
+                        ChangeDeviceState(shutdown ? DeviceStateTransition::ResetTask
+                                                    : DeviceStateTransition::Run);
+                        break;
+                    }
+                    case DeviceState::Running: {
+                        ChangeDeviceState(DeviceStateTransition::Stop);
+                        break;
+                    }
+                    case DeviceState::Exiting: {
+                        ReleaseDeviceControl();
+                        break;
+                    }
+                    default:
+                        break;
+                }
+            } catch (fair::mq::PluginServices::DeviceControlError const&) {
+                // this means we do not have device control
+            }
+        });
+    }
+
+    ~MyController() override { ReleaseDeviceControl(); }
+};
+
+// auto MyControllerProgramOptions() -> fair::mq::Plugin::ProgOptions
+// {
+// auto plugin_options = boost::program_options::options_description{"MyController Plugin"};
+// plugin_options.add_options()
+// ("custom-dummy-option", boost::program_options::value<std::string>(), "Cool custom option.")
+// ("custom-dummy-option2", boost::program_options::value<std::string>(), "Another one.");
+// return plugin_options;
+// }
+
+}   // namespace example
+
+REGISTER_FAIRMQ_PLUGIN(example::MyController,   // Class name
+                       mycontroller,            // Plugin name (string, lower case chars only)
+                       (fair::mq::Plugin::Version{0, 42, 0}),     // Version
+                       "Mr. Dummy <dummy@test.net>",              // Maintainer
+                       "https://git.test.net/mycontroller.git",   // Homepage
+                       //                       example::MyControllerProgramOptions   // Free
+                       //                       function which declares custom
+                       // program options for the plugin
+                       fair::mq::Plugin::NoProgramOptions)
+
+#endif /* FAIR_MQ_EXAMPLE_CUSTOM_CONTROLLER_PLUGIN */

examples/custom-controller/MyDevice.h

@@ -0,0 +1,21 @@
+/********************************************************************************
+ *    Copyright (C) 2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#ifndef FAIR_MQ_EXAMPLE_CUSTOM_CONTROLLER_DEVICE
+#define FAIR_MQ_EXAMPLE_CUSTOM_CONTROLLER_DEVICE
+
+#include <fairmq/Device.h>
+
+namespace example {
+
+using MyDevice = fair::mq::Device;
+
+}   // namespace example
+
+#endif /* FAIR_MQ_EXAMPLE_CUSTOM_CONTROLLER_DEVICE */
+

examples/custom-controller/main.cxx

@@ -0,0 +1,46 @@
+/********************************************************************************
+ *    Copyright (C) 2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include "MyController.h"
+#include "MyDevice.h"
+
+#include <chrono>   // for std::chrono_literals
+#include <fairlogger/Logger.h>
+#include <fairmq/DeviceRunner.h>
+#include <memory>   // for std::make_unique
+
+int main(int argc, char* argv[])
+{
+    using namespace fair::mq;
+    using namespace std::chrono_literals;
+
+    DeviceRunner runner(argc, argv);
+
+    runner.AddHook<hooks::LoadPlugins>([](DeviceRunner& r) {
+        r.fPluginManager.LoadPlugin("s:mycontroller");
+        // 's:' stands for static because the plugin is compiled into the executable
+        // 'mycontroller' is the plugin name passed as second arg to REGISTER_FAIRMQ_PLUGIN
+    });
+
+    fair::Logger::SetConsoleSeverity(fair::Severity::debug);
+
+    runner.AddHook<hooks::InstantiateDevice>([](DeviceRunner& r) {
+        r.fConfig.SetProperty<int>("catch-signals", 0);
+        r.fConfig.SetProperty<bool>("please-shut-me-down", false);
+        r.fDevice = std::make_unique<example::MyDevice>(r.fConfig);
+
+        r.fDevice->SubscribeToStateChange("example", [&r](auto state) {
+            if (state == State::Running) {
+                r.fDevice->WaitFor(3s);
+                r.fConfig.SetProperty<bool>("please-shut-me-down", true);
+            }
+        });
+    });
+
+    return runner.RunWithExceptionHandlers();
+}

examples/multipart/CMakeLists.txt

@@ -1,56 +1,11 @@
- ################################################################################
- #    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
- #                                                                              #
- #              This software is distributed under the terms of the             #
- #              GNU Lesser General Public Licence (LGPL) version 3,             #
- #                  copied verbatim in the file "LICENSE"                       #
- ################################################################################
+################################################################################
+# Copyright (C) 2014-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
 
-add_executable(fairmq-ex-multipart-sampler sampler.cxx)
-target_link_libraries(fairmq-ex-multipart-sampler PRIVATE FairMQ)
-
-add_executable(fairmq-ex-multipart-sink sink.cxx)
-target_link_libraries(fairmq-ex-multipart-sink PRIVATE FairMQ)
-
-add_custom_target(ExampleMultipart DEPENDS fairmq-ex-multipart-sampler fairmq-ex-multipart-sink)
-
-set(EX_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_BINARY_DIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multipart.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-multipart.sh)
-
-# test
-
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multipart.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh)
-
-add_test(NAME Example.Multipart.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh zeromq)
-set_tests_properties(Example.Multipart.zeromq PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Received message with 7 parts")
-
-add_test(NAME Example.Multipart.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh shmem)
-set_tests_properties(Example.Multipart.shmem PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Received message with 7 parts")
-
-if(BUILD_OFI_TRANSPORT)
-  add_test(NAME Example.Multipart.ofi COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multipart.sh ofi)
-  set_tests_properties(Example.Multipart.ofi PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Received message with 7 parts")
-endif()
-
-# install
-
-install(
-  TARGETS
-  fairmq-ex-multipart-sampler
-  fairmq-ex-multipart-sink
-
-  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
-  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
-)
-
-# configure run script with different executable paths for build and for install directories
-set(EX_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multipart.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-multipart.sh_install)
-
-install(
-  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-multipart.sh_install
-  DESTINATION ${PROJECT_INSTALL_BINDIR}
-  RENAME fairmq-start-ex-multipart.sh
+add_example(NAME multipart
+  DEVICE sampler sink
 )

examples/multipart/sink.cxx

@@ -24,6 +24,10 @@ struct Sink : fair::mq::Device
     {
         LOG(info) << "Received message with " << parts.Size() << " parts";
 
+        if (parts.Size() != 7) {
+            throw std::runtime_error("Number of received parts != 7");
+        }
+
         example_multipart::Header header;
         header.stopFlag = (static_cast<example_multipart::Header*>(parts.At(0)->GetData()))->stopFlag;
 

examples/multipart/test-ex-multipart.sh.in

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
 
 transport="zeromq"
@@ -12,14 +14,10 @@ session="$(@CMAKE_BINARY_DIR@/fairmq/fairmq-uuid-gen -h)"
 chan="data"
 chanAddr=""
 chanIpcFile="/tmp/fmq_$session""_""$chan""_""$transport"
-if [ $transport = "ofi" ]; then
-    chanAddr="tcp://127.0.0.1:5656"
-else
-    chanAddr="ipc://""$chanIpcFile"
-fi
+chanAddr="ipc://""$chanIpcFile"
 
 # setup a trap to kill everything if the test fails/timeouts
-trap 'kill -TERM $SAMPLER_PID; kill -TERM $SINK_PID; wait $SAMPLER_PID; wait $SINK_PID; rm $chanIpcFile' TERM
+trap 'set +e; kill -TERM $SAMPLER_PID; kill -TERM $SINK_PID; wait $SAMPLER_PID; wait $SINK_PID; rm $chanIpcFile; exit 0' TERM
 
 SAMPLER="fairmq-ex-multipart-sampler"
 SAMPLER+=" --id sampler1"
@@ -27,6 +25,7 @@ SAMPLER+=" --transport $transport"
 SAMPLER+=" --verbosity veryhigh"
 SAMPLER+=" --session $session"
 SAMPLER+=" --shm-segment-size 100000000"
+SAMPLER+=" --shm-monitor true"
 SAMPLER+=" --max-iterations 1"
 SAMPLER+=" --control static --color false"
 SAMPLER+=" --channel-config name=$chan,type=pair,method=connect,rateLogging=0,address=$chanAddr,linger=1000"
@@ -39,6 +38,7 @@ SINK+=" --transport $transport"
 SINK+=" --verbosity veryhigh"
 SINK+=" --session $session"
 SINK+=" --shm-segment-size 100000000"
+SINK+=" --shm-monitor true"
 SINK+=" --control static --color false"
 SINK+=" --channel-config name=$chan,type=pair,method=bind,rateLogging=0,address=$chanAddr"
 @CMAKE_CURRENT_BINARY_DIR@/$SINK &
@@ -47,4 +47,6 @@ SINK_PID=$!
 wait $SAMPLER_PID
 wait $SINK_PID
 
+set +e
 rm $chanIpcFile
+exit 0

examples/multiple-channels/CMakeLists.txt

@@ -1,52 +1,12 @@
- ################################################################################
- #    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
- #                                                                              #
- #              This software is distributed under the terms of the             #
- #              GNU Lesser General Public Licence (LGPL) version 3,             #
- #                  copied verbatim in the file "LICENSE"                       #
- ################################################################################
+################################################################################
+# Copyright (C) 2014-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
 
-add_executable(fairmq-ex-multiple-channels-sampler sampler.cxx)
-target_link_libraries(fairmq-ex-multiple-channels-sampler PRIVATE FairMQ)
-
-add_executable(fairmq-ex-multiple-channels-broadcaster broadcaster.cxx)
-target_link_libraries(fairmq-ex-multiple-channels-broadcaster PRIVATE FairMQ)
-
-add_executable(fairmq-ex-multiple-channels-sink sink.cxx)
-target_link_libraries(fairmq-ex-multiple-channels-sink PRIVATE FairMQ)
-
-add_custom_target(ExampleMultipleChannels DEPENDS fairmq-ex-multiple-channels-sampler fairmq-ex-multiple-channels-broadcaster fairmq-ex-multiple-channels-sink)
-
-set(EX_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_BINARY_DIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multiple-channels.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-multiple-channels.sh)
-
-# test
-
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multiple-channels.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh)
-
-add_test(NAME Example.MultipleChannels.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-channels.sh zeromq)
-set_tests_properties(Example.MultipleChannels.zeromq PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Received messages from both sources.")
-
-# install
-
-install(
-  TARGETS
-  fairmq-ex-multiple-channels-sampler
-  fairmq-ex-multiple-channels-broadcaster
-  fairmq-ex-multiple-channels-sink
-
-  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
-  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
-)
-
-# configure run script with different executable paths for build and for install directories
-set(EX_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multiple-channels.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-multiple-channels.sh_install)
-
-install(
-  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-multiple-channels.sh_install
-  DESTINATION ${PROJECT_INSTALL_BINDIR}
-  RENAME fairmq-start-ex-multiple-channels.sh
+add_example(NAME multiple-channels
+  DEVICE sampler broadcaster sink
+  TRANSPORT zeromq
 )

examples/multiple-channels/test-ex-multiple-channels.sh.in

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
 
 transport="zeromq"
@@ -15,7 +17,7 @@ dataChanAddr="/tmp/fmq_$session""_""$dataChan""_""$transport"
 broadcastChanAddr="/tmp/fmq_$session""_""$broadcastChan""_""$transport"
 
 # setup a trap to kill everything if the test fails/timeouts
-trap 'kill -TERM $SAMPLER_PID; kill -TERM $SINK_PID; kill -TERM $BROADCASTER_PID; wait $SAMPLER_PID; wait $SINK_PID; wait $BROADCASTER_PID; rm $dataChanAddr; rm $broadcastChanAddr' TERM
+trap 'set +e; kill -TERM $SAMPLER_PID; kill -TERM $SINK_PID; kill -TERM $BROADCASTER_PID; wait $SAMPLER_PID; wait $SINK_PID; wait $BROADCASTER_PID; rm $dataChanAddr; rm $broadcastChanAddr; exit 0' TERM
 
 SINK="fairmq-ex-multiple-channels-sink"
 SINK+=" --id sink1"
@@ -23,6 +25,7 @@ SINK+=" --session $session"
 SINK+=" --transport $transport"
 SINK+=" --verbosity veryhigh --severity debug"
 SINK+=" --shm-segment-size 100000000"
+SINK+=" --shm-monitor true"
 SINK+=" --max-iterations 1"
 SINK+=" --control static --color false"
 SINK+=" --channel-config name=$dataChan,type=pull,method=connect,rateLogging=0,address=ipc://$dataChanAddr"
@@ -38,6 +41,7 @@ SAMPLER+=" --session $session"
 SAMPLER+=" --transport $transport"
 SAMPLER+=" --verbosity veryhigh --severity debug"
 SAMPLER+=" --shm-segment-size 100000000"
+SAMPLER+=" --shm-monitor true"
 SAMPLER+=" --max-iterations 1"
 SAMPLER+=" --control static --color false"
 SAMPLER+=" --channel-config name=$dataChan,type=push,method=bind,rateLogging=0,address=ipc://$dataChanAddr"
@@ -51,6 +55,7 @@ BROADCASTER+=" --session $session"
 BROADCASTER+=" --transport $transport"
 BROADCASTER+=" --verbosity veryhigh --severity debug"
 BROADCASTER+=" --shm-segment-size 100000000"
+BROADCASTER+=" --shm-monitor true"
 BROADCASTER+=" --control static --color false"
 BROADCASTER+=" --channel-config name=$broadcastChan,type=pub,method=bind,rateLogging=0,address=ipc://$broadcastChanAddr"
 @CMAKE_CURRENT_BINARY_DIR@/$BROADCASTER &
@@ -65,4 +70,6 @@ kill -SIGINT $BROADCASTER_PID
 # wait for broadcaster to finish
 wait $BROADCASTER_PID
 
+set +e
 rm $dataChanAddr; rm $broadcastChanAddr
+exit 0

examples/multiple-transports/CMakeLists.txt

@@ -1,51 +1,12 @@
- ################################################################################
- #    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
- #                                                                              #
- #              This software is distributed under the terms of the             #
- #              GNU Lesser General Public Licence (LGPL) version 3,             #
- #                  copied verbatim in the file "LICENSE"                       #
- ################################################################################
+################################################################################
+# Copyright (C) 2014-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
 
-add_executable(fairmq-ex-multiple-transports-sampler1 sampler1.cxx)
-target_link_libraries(fairmq-ex-multiple-transports-sampler1 PRIVATE FairMQ)
-
-add_executable(fairmq-ex-multiple-transports-sampler2 sampler2.cxx)
-target_link_libraries(fairmq-ex-multiple-transports-sampler2 PRIVATE FairMQ)
-
-add_executable(fairmq-ex-multiple-transports-sink sink.cxx)
-target_link_libraries(fairmq-ex-multiple-transports-sink PRIVATE FairMQ)
-
-add_custom_target(ExampleMultipleTransports DEPENDS fairmq-ex-multiple-transports-sampler1 fairmq-ex-multiple-transports-sampler2 fairmq-ex-multiple-transports-sink)
-
-set(EX_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_BINARY_DIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multiple-transports.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-multiple-transports.sh)
-
-# test
-
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-multiple-transports.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-transports.sh)
-add_test(NAME Example.MultipleTransports COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-multiple-transports.sh)
-set_tests_properties(Example.MultipleTransports PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Received messages from both sources.")
-
-# install
-
-install(
-  TARGETS
-  fairmq-ex-multiple-transports-sampler1
-  fairmq-ex-multiple-transports-sampler2
-  fairmq-ex-multiple-transports-sink
-
-  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
-  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
-)
-
-# configure run script with different executable paths for install directories
-set(EX_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-multiple-transports.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-multiple-transports.sh_install)
-
-install(
-  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-multiple-transports.sh_install
-  DESTINATION ${PROJECT_INSTALL_BINDIR}
-  RENAME fairmq-start-ex-multiple-transports.sh
+add_example(NAME multiple-transports
+  DEVICE sampler1 sampler2 sink
+  NO_TRANSPORT
 )

examples/multiple-transports/sink.cxx

@@ -9,6 +9,8 @@
 #include <fairmq/Device.h>
 #include <fairmq/runDevice.h>
 
+#include <stdexcept>
+
 namespace bpo = boost::program_options;
 
 struct Sink : fair::mq::Device
@@ -32,7 +34,7 @@ struct Sink : fair::mq::Device
         // Creates a message using the transport of channel ack
         fair::mq::MessagePtr ack(NewMessageFor("ack", 0));
         if (Send(ack, "ack") < 0) {
-            return false;
+            throw std::runtime_error("could not send an ack");
         }
 
         // return true if want to be called again (otherwise go to IDLE state)

examples/multiple-transports/test-ex-multiple-transports.sh.in

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
 
 session="$(@CMAKE_BINARY_DIR@/fairmq/fairmq-uuid-gen -h)"
@@ -10,12 +12,13 @@ chan1Addr="/tmp/fmq_$session""_""$chan1"
 chan2Addr="/tmp/fmq_$session""_""$chan2"
 ackChanAddr="/tmp/fmq_$session""_""$ackChan"
 
-trap 'kill -TERM $SAMPLER1_PID; kill -TERM $SAMPLER2_PID; kill -TERM $SINK_PID; wait $SAMPLER1_PID; wait $SAMPLER2_PID; wait $SINK_PID; @CMAKE_BINARY_DIR@/fairmq/fairmq-shmmonitor --cleanup --session $SESSION; rm $chan1Addr; rm $chan2Addr; rm $ackChanAddr' TERM
+trap 'set +e; kill -TERM $SAMPLER1_PID; kill -TERM $SAMPLER2_PID; kill -TERM $SINK_PID; wait $SAMPLER1_PID; wait $SAMPLER2_PID; wait $SINK_PID; @CMAKE_BINARY_DIR@/fairmq/fairmq-shmmonitor --cleanup --session $SESSION; rm $chan1Addr; rm $chan2Addr; rm $ackChanAddr; exit 0' TERM
 
 SINK="fairmq-ex-multiple-transports-sink"
 SINK+=" --id sink1"
 SINK+=" --verbosity veryhigh --severity debug"
 SINK+=" --shm-segment-size 100000000"
+SINK+=" --shm-monitor true"
 SINK+=" --session $session"
 SINK+=" --max-iterations 1"
 SINK+=" --control static --color false"
@@ -31,6 +34,7 @@ SAMPLER1+=" --id sampler1"
 SAMPLER1+=" --session $session"
 SAMPLER1+=" --verbosity veryhigh --severity debug"
 SAMPLER1+=" --shm-segment-size 100000000"
+SAMPLER1+=" --shm-monitor true"
 SAMPLER1+=" --max-iterations 1"
 SAMPLER1+=" --control static --color false"
 SAMPLER1+=" --transport shmem"
@@ -44,6 +48,7 @@ SAMPLER2+=" --id sampler2"
 SAMPLER2+=" --session $session"
 SAMPLER2+=" --verbosity veryhigh --severity debug"
 SAMPLER2+=" --shm-segment-size 100000000"
+SAMPLER2+=" --shm-monitor true"
 SAMPLER2+=" --max-iterations 1"
 SAMPLER2+=" --control static --color false"
 SAMPLER2+=" --transport zeromq"
@@ -55,4 +60,6 @@ wait $SAMPLER1_PID
 wait $SAMPLER2_PID
 wait $SINK_PID
 
+set +e
 rm $chan1Addr; rm $chan2Addr; rm $ackChanAddr
+exit 0

examples/n-m/receiver.cxx

@@ -1,5 +1,5 @@
 /********************************************************************************
- * Copyright (C) 2020-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
+ * Copyright (C) 2020-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -54,7 +54,7 @@ struct Receiver : fair::mq::Device
                 fBuffer[h.id].start = chrono::steady_clock::now();
             }
             // if the received ID has not previously been discarded, store the data part in the buffer
-            fBuffer[h.id].parts.AddPart(move(parts.At(1)));
+            fBuffer[h.id].parts.AddPart(std::move(parts.At(1)));
         } else {
             // if received ID has been previously discarded.
             LOG(debug) << "Received part from an already discarded timeframe with id " << h.id;

examples/readout/CMakeLists.txt

@@ -1,59 +1,13 @@
- ################################################################################
- #    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
- #                                                                              #
- #              This software is distributed under the terms of the             #
- #              GNU Lesser General Public Licence (LGPL) version 3,             #
- #                  copied verbatim in the file "LICENSE"                       #
- ################################################################################
+################################################################################
+# Copyright (C) 2014-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
 
-add_executable(fairmq-ex-readout-readout readout.cxx)
-target_link_libraries(fairmq-ex-readout-readout PRIVATE FairMQ)
-
-add_executable(fairmq-ex-readout-builder builder.cxx)
-target_link_libraries(fairmq-ex-readout-builder PRIVATE FairMQ)
-
-add_executable(fairmq-ex-readout-processor processor.cxx)
-target_link_libraries(fairmq-ex-readout-processor PRIVATE FairMQ)
-
-add_executable(fairmq-ex-readout-sender sender.cxx)
-target_link_libraries(fairmq-ex-readout-sender PRIVATE FairMQ)
-
-add_executable(fairmq-ex-readout-receiver receiver.cxx)
-target_link_libraries(fairmq-ex-readout-receiver PRIVATE FairMQ)
-
-set(EX_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_BINARY_DIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-readout.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout.sh)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-readout-processing.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout-processing.sh)
-
-# install
-
-install(
-  TARGETS
-  fairmq-ex-readout-readout
-  fairmq-ex-readout-builder
-  fairmq-ex-readout-processor
-  fairmq-ex-readout-sender
-  fairmq-ex-readout-receiver
-
-  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
-  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
-)
-
-# configure run script with different executable paths for build and for install directories
-set(EX_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-readout.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout.sh_install)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-readout-processing.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout-processing.sh_install)
-
-install(
-  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout.sh_install
-  DESTINATION ${PROJECT_INSTALL_BINDIR}
-  RENAME fairmq-start-ex-readout.sh
-)
-
-install(
-  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-readout-processing.sh_install
-  DESTINATION ${PROJECT_INSTALL_BINDIR}
-  RENAME fairmq-start-ex-readout-processing.sh
+add_example(NAME readout
+  DEVICE readout builder processor sender receiver
+  SCRIPT readout readout-processing
+  NO_TEST
 )

examples/readout/README.md

@@ -7,19 +7,19 @@ This examples shows two possible topologies (out of many) for a node connected t
 ```
 |------------------------------- Readout Node ---------------------------|     |- Processing Node -|
 | Readout            -->            Builder          -->          Sender | --> |     Receiver      |
-| [# shared memory segment (unused in this topology) ##################] | ofi |                   |
+| [# shared memory segment (unused in this topology) ##################] | zmq |                   |
 | [# shmem unmanaged region (readout writes here, others read) ########] |     |                   |
 |------------------------------------------------------------------------|     |-------------------|
 ```
 
-The devices one the Readout Node communicate via shared memory transport. Readout device writes into shared memory unmanaged region. The data is then forwarded through Builder to Sender process, which sends it out via OFI transport.
+The devices one the Readout Node communicate via shared memory transport. Readout device writes into shared memory unmanaged region. The data is then forwarded through Builder to Sender process, which sends it out via zeromq transport.
 
 ## Setup with generating new data on the Readout node
 
 ```
 |------------------------------- Readout Node ---------------------------|     |- Processing Node -|
 | Readout     -->     Builder      -->      Processor     -->     Sender | --> |     Receiver      |
-| [# shared memory segment (used between Proccessor and Sender) #######] | ofi |                   |
+| [# shared memory segment (used between Proccessor and Sender) #######] | zmq |                   |
 | [# shmem unmanaged region (readout writes here, builder & proc read) ] |     |                   |
 |------------------------------------------------------------------------|     |-------------------|
 ```

examples/readout/fairmq-start-ex-readout-processing.sh.in

@@ -31,12 +31,10 @@ SENDER="fairmq-ex-readout-sender"
 SENDER+=" --id sender1"
 SENDER+=" --input-name ps"
 SENDER+=" --channel-config name=ps,type=pair,method=bind,address=tcp://localhost:7779,transport=shmem"
-#SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7780,transport=ofi"
 SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7780,transport=zeromq"
 xterm -geometry 80x23+1000+0 -hold -e @EX_BIN_DIR@/$SENDER &
 
 RECEIVER="fairmq-ex-readout-receiver"
 RECEIVER+=" --id receiver1"
-#RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7780,transport=ofi"
 RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7780,transport=zeromq"
 xterm -geometry 80x23+1500+0 -hold -e @EX_BIN_DIR@/$RECEIVER &

examples/readout/fairmq-start-ex-readout.sh.in

@@ -25,12 +25,10 @@ SENDER="fairmq-ex-readout-sender"
 SENDER+=" --id sender1"
 SENDER+=" --input-name bs"
 SENDER+=" --channel-config name=bs,type=pair,method=bind,address=tcp://localhost:7778,transport=shmem"
-# SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7779,transport=ofi"
 SENDER+="                  name=sr,type=pair,method=connect,address=tcp://localhost:7779,transport=zeromq"
 xterm -geometry 80x23+1000+0 -hold -e @EX_BIN_DIR@/$SENDER &
 
 RECEIVER="fairmq-ex-readout-receiver"
 RECEIVER+=" --id receiver1"
-# RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7779,transport=ofi"
 RECEIVER+=" --channel-config name=sr,type=pair,method=bind,address=tcp://localhost:7779,transport=zeromq"
 xterm -geometry 80x23+1500+0 -hold -e @EX_BIN_DIR@/$RECEIVER &

examples/region/CMakeLists.txt

@@ -1,54 +1,12 @@
- ################################################################################
- #    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
- #                                                                              #
- #              This software is distributed under the terms of the             #
- #              GNU Lesser General Public Licence (LGPL) version 3,             #
- #                  copied verbatim in the file "LICENSE"                       #
- ################################################################################
+################################################################################
+# Copyright (C) 2014-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
 
-add_executable(fairmq-ex-region-sampler sampler.cxx)
-target_link_libraries(fairmq-ex-region-sampler PRIVATE FairMQ)
-
-add_executable(fairmq-ex-region-sink sink.cxx)
-target_link_libraries(fairmq-ex-region-sink PRIVATE FairMQ)
-
-add_executable(fairmq-ex-region-keep-alive keep-alive.cxx)
-target_link_libraries(fairmq-ex-region-keep-alive PRIVATE FairMQ)
-
-add_custom_target(ExampleRegion DEPENDS fairmq-ex-region-sampler fairmq-ex-region-sink fairmq-ex-region-keep-alive)
-
-set(EX_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_BINARY_DIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-region.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-region.sh)
-
-# test
-
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-region.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh)
-
-add_test(NAME Example.Region.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh zeromq)
-set_tests_properties(Example.Region.zeromq PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Received [0-9*] acks")
-
-add_test(NAME Example.Region.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-region.sh shmem)
-set_tests_properties(Example.Region.shmem PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Received [0-9*] acks")
-
-# install
-
-install(
-  TARGETS
-  fairmq-ex-region-sampler
-  fairmq-ex-region-sink
-
-  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
-  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
-)
-
-# configure run script with different executable paths for build and for install directories
-set(EX_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-region.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-region.sh_install)
-
-install(
-  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-region.sh_install
-  DESTINATION ${PROJECT_INSTALL_BINDIR}
-  RENAME fairmq-start-ex-region.sh
+add_example(NAME region
+  DEVICE sampler processor sink keep-alive
+  SCRIPT region region-advanced
 )

examples/region/fairmq-start-ex-region-advanced-external.sh.in

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
+
+transport=${1:-shmem}
+msgSize=${2:-1000000}
+
+SAMPLER="fairmq-ex-region-sampler"
+SAMPLER+=" --id sampler1"
+# SAMPLER+=" --sampling-rate 10"
+SAMPLER+=" --severity debug"
+SAMPLER+=" --msg-size $msgSize"
+SAMPLER+=" --transport $transport"
+SAMPLER+=" --shmid 1"
+SAMPLER+=" --shm-monitor false"
+SAMPLER+=" --rc-segment-size 200000000"
+SAMPLER+=" --external-region true"
+SAMPLER+=" --shm-no-cleanup true"
+SAMPLER+=" --chan-name data1"
+SAMPLER+=" --channel-config name=data1,type=push,method=bind,address=tcp://127.0.0.1:7777"
+xterm -geometry 90x60+0+0 -hold -e @EX_BIN_DIR@/$SAMPLER &
+
+PROCESSOR1="fairmq-ex-region-processor"
+PROCESSOR1+=" --id processor1"
+PROCESSOR1+=" --severity debug"
+PROCESSOR1+=" --transport $transport"
+PROCESSOR1+=" --shmid 1"
+PROCESSOR1+=" --shm-segment-id 1"
+PROCESSOR1+=" --shm-monitor false"
+PROCESSOR1+=" --shm-no-cleanup true"
+PROCESSOR1+=" --channel-config name=data1,type=pull,method=connect,address=tcp://127.0.0.1:7777"
+PROCESSOR1+="                  name=data2,type=push,method=bind,address=tcp://127.0.0.1:7778"
+PROCESSOR1+="                  name=data3,type=push,method=bind,address=tcp://127.0.0.1:7779"
+xterm -geometry 90x40+550+40 -hold -e @EX_BIN_DIR@/$PROCESSOR1 &
+
+PROCESSOR2="fairmq-ex-region-processor"
+PROCESSOR2+=" --id processor2"
+PROCESSOR2+=" --severity debug"
+PROCESSOR2+=" --transport $transport"
+PROCESSOR2+=" --shmid 1"
+PROCESSOR2+=" --shm-segment-id 2"
+PROCESSOR2+=" --shm-monitor false"
+PROCESSOR2+=" --shm-no-cleanup true"
+PROCESSOR2+=" --channel-config name=data1,type=pull,method=connect,address=tcp://127.0.0.1:7777"
+PROCESSOR2+="                  name=data2,type=push,method=bind,address=tcp://127.0.0.1:7788"
+PROCESSOR2+="                  name=data3,type=push,method=bind,address=tcp://127.0.0.1:7789"
+xterm -geometry 90x40+550+600 -hold -e @EX_BIN_DIR@/$PROCESSOR2 &
+
+SINK1_1="fairmq-ex-region-sink"
+SINK1_1+=" --id sink1_1"
+SINK1_1+=" --severity debug"
+SINK1_1+=" --chan-name data2"
+SINK1_1+=" --transport $transport"
+SINK1_1+=" --shmid 1"
+SINK1_1+=" --shm-segment-id 1"
+SINK1_1+=" --shm-monitor false"
+SINK1_1+=" --shm-no-cleanup true"
+SINK1_1+=" --channel-config name=data2,type=pull,method=connect,address=tcp://127.0.0.1:7778"
+xterm -geometry 90x20+1100+0 -hold -e @EX_BIN_DIR@/$SINK1_1 &
+
+SINK1_2="fairmq-ex-region-sink"
+SINK1_2+=" --id sink1_2"
+SINK1_2+=" --severity debug"
+SINK1_2+=" --chan-name data3"
+SINK1_2+=" --transport $transport"
+SINK1_2+=" --shmid 1"
+SINK1_2+=" --shm-segment-id 1"
+SINK1_2+=" --shm-monitor false"
+SINK1_2+=" --shm-no-cleanup true"
+SINK1_2+=" --channel-config name=data3,type=pull,method=connect,address=tcp://127.0.0.1:7779"
+xterm -geometry 90x20+1100+300 -hold -e @EX_BIN_DIR@/$SINK1_2 &
+
+SINK2_1="fairmq-ex-region-sink"
+SINK2_1+=" --id sink2_1"
+SINK2_1+=" --severity debug"
+SINK2_1+=" --chan-name data2"
+SINK2_1+=" --transport $transport"
+SINK2_1+=" --shmid 1"
+SINK2_1+=" --shm-segment-id 2"
+SINK2_1+=" --shm-monitor false"
+SINK2_1+=" --shm-no-cleanup true"
+SINK2_1+=" --channel-config name=data2,type=pull,method=connect,address=tcp://127.0.0.1:7788"
+xterm -geometry 90x20+1100+600 -hold -e @EX_BIN_DIR@/$SINK2_1 &
+
+SINK2_2="fairmq-ex-region-sink"
+SINK2_2+=" --id sink2_2"
+SINK2_2+=" --severity debug"
+SINK2_2+=" --chan-name data3"
+SINK2_2+=" --transport $transport"
+SINK2_2+=" --shmid 1"
+SINK2_2+=" --shm-segment-id 2"
+SINK2_2+=" --shm-monitor false"
+SINK2_2+=" --shm-no-cleanup true"
+SINK2_2+=" --channel-config name=data3,type=pull,method=connect,address=tcp://127.0.0.1:7789"
+xterm -geometry 90x20+1100+900 -hold -e @EX_BIN_DIR@/$SINK2_2 &

examples/region/fairmq-start-ex-region-advanced.sh.in

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
+
+transport=${1:-shmem}
+msgSize=${2:-1000000}
+
+SAMPLER="fairmq-ex-region-sampler"
+SAMPLER+=" --id sampler1"
+# SAMPLER+=" --sampling-rate 10"
+SAMPLER+=" --severity debug"
+SAMPLER+=" --msg-size $msgSize"
+SAMPLER+=" --transport $transport"
+SAMPLER+=" --shm-monitor true"
+SAMPLER+=" --chan-name data1"
+SAMPLER+=" --channel-config name=data1,type=push,method=bind,address=tcp://127.0.0.1:7777"
+xterm -geometry 90x60+0+0 -hold -e @EX_BIN_DIR@/$SAMPLER &
+
+PROCESSOR1="fairmq-ex-region-processor"
+PROCESSOR1+=" --id processor1"
+PROCESSOR1+=" --severity debug"
+PROCESSOR1+=" --transport $transport"
+PROCESSOR1+=" --shm-segment-id 1"
+PROCESSOR1+=" --shm-monitor true"
+PROCESSOR1+=" --channel-config name=data1,type=pull,method=connect,address=tcp://127.0.0.1:7777"
+PROCESSOR1+="                  name=data2,type=push,method=bind,address=tcp://127.0.0.1:7778"
+PROCESSOR1+="                  name=data3,type=push,method=bind,address=tcp://127.0.0.1:7779"
+xterm -geometry 90x40+550+40 -hold -e @EX_BIN_DIR@/$PROCESSOR1 &
+
+PROCESSOR2="fairmq-ex-region-processor"
+PROCESSOR2+=" --id processor2"
+PROCESSOR2+=" --severity debug"
+PROCESSOR2+=" --transport $transport"
+PROCESSOR2+=" --shm-segment-id 2"
+PROCESSOR2+=" --shm-monitor true"
+PROCESSOR2+=" --channel-config name=data1,type=pull,method=connect,address=tcp://127.0.0.1:7777"
+PROCESSOR2+="                  name=data2,type=push,method=bind,address=tcp://127.0.0.1:7788"
+PROCESSOR2+="                  name=data3,type=push,method=bind,address=tcp://127.0.0.1:7789"
+xterm -geometry 90x40+550+600 -hold -e @EX_BIN_DIR@/$PROCESSOR2 &
+
+SINK1_1="fairmq-ex-region-sink"
+SINK1_1+=" --id sink1_1"
+SINK1_1+=" --severity debug"
+SINK1_1+=" --chan-name data2"
+SINK1_1+=" --transport $transport"
+SINK1_1+=" --shm-segment-id 1"
+SINK1_1+=" --shm-monitor true"
+SINK1_1+=" --channel-config name=data2,type=pull,method=connect,address=tcp://127.0.0.1:7778"
+xterm -geometry 90x20+1100+0 -hold -e @EX_BIN_DIR@/$SINK1_1 &
+
+SINK1_2="fairmq-ex-region-sink"
+SINK1_2+=" --id sink1_2"
+SINK1_2+=" --severity debug"
+SINK1_2+=" --chan-name data3"
+SINK1_2+=" --transport $transport"
+SINK1_2+=" --shm-segment-id 1"
+SINK1_2+=" --shm-monitor true"
+SINK1_2+=" --channel-config name=data3,type=pull,method=connect,address=tcp://127.0.0.1:7779"
+xterm -geometry 90x20+1100+300 -hold -e @EX_BIN_DIR@/$SINK1_2 &
+
+SINK2_1="fairmq-ex-region-sink"
+SINK2_1+=" --id sink2_1"
+SINK2_1+=" --severity debug"
+SINK2_1+=" --chan-name data2"
+SINK2_1+=" --transport $transport"
+SINK2_1+=" --shm-segment-id 2"
+SINK2_1+=" --shm-monitor true"
+SINK2_1+=" --channel-config name=data2,type=pull,method=connect,address=tcp://127.0.0.1:7788"
+xterm -geometry 90x20+1100+600 -hold -e @EX_BIN_DIR@/$SINK2_1 &
+
+SINK2_2="fairmq-ex-region-sink"
+SINK2_2+=" --id sink2_2"
+SINK2_2+=" --severity debug"
+SINK2_2+=" --chan-name data3"
+SINK2_2+=" --transport $transport"
+SINK2_2+=" --shm-segment-id 2"
+SINK2_2+=" --shm-monitor true"
+SINK2_2+=" --channel-config name=data3,type=pull,method=connect,address=tcp://127.0.0.1:7789"
+xterm -geometry 90x20+1100+900 -hold -e @EX_BIN_DIR@/$SINK2_2 &

examples/region/fairmq-start-ex-region.sh.in

@@ -2,16 +2,8 @@
 
 export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
 
-transport="shmem"
-msgSize="1000000"
-
-if [[ $1 =~ ^[a-z]+$ ]]; then
-    transport=$1
-fi
-
-if [[ $2 =~ ^[0-9]+$ ]]; then
-    msgSize=$1
-fi
+transport=${1:-shmem}
+msgSize=${2:-1000000}
 
 SAMPLER="fairmq-ex-region-sampler"
 SAMPLER+=" --id sampler1"
@@ -19,6 +11,10 @@ SAMPLER+=" --severity debug"
 SAMPLER+=" --msg-size $msgSize"
 # SAMPLER+=" --rate 10"
 SAMPLER+=" --transport $transport"
+# SAMPLER+=" --external-region true"
+# SAMPLER+=" --shm-no-cleaup true"
+SAMPLER+=" --shm-monitor true"
+# SAMPLER+=" --shmid 1"
 SAMPLER+=" --channel-config name=data,type=push,method=bind,address=tcp://127.0.0.1:7777,sndKernelSize=212992"
 xterm -geometry 120x60+0+0 -hold -e @EX_BIN_DIR@/$SAMPLER &
 
@@ -26,5 +22,8 @@ SINK="fairmq-ex-region-sink"
 SINK+=" --id sink1"
 SINK+=" --severity debug"
 SINK+=" --transport $transport"
+# SINK+=" --shm-no-cleaup true"
+SINK+=" --shm-monitor true"
+# SINK+=" --shmid 1"
 SINK+=" --channel-config name=data,type=pull,method=connect,address=tcp://127.0.0.1:7777,rcvKernelSize=212992"
 xterm -geometry 120x60+750+0 -hold -e @EX_BIN_DIR@/$SINK &

examples/region/keep-alive.cxx

@@ -6,10 +6,9 @@
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 #include <fairmq/shmem/Common.h>
-#include <fairmq/shmem/UnmanagedRegion.h>
-#include <fairmq/shmem/Segment.h>
 #include <fairmq/shmem/Monitor.h>
-
+#include <fairmq/shmem/Segment.h>
+#include <fairmq/shmem/UnmanagedRegion.h>
 #include <fairmq/tools/Unique.h>
 
 #include <fairlogger/Logger.h>
@@ -17,75 +16,129 @@
 #include <boost/algorithm/string.hpp>
 #include <boost/program_options.hpp>
 
-#include <csignal>
-
 #include <chrono>
+#include <csignal>
 #include <map>
+#include <mutex>
 #include <string>
 #include <thread>
 
 using namespace std;
 using namespace boost::program_options;
 
-namespace
-{
-    volatile sig_atomic_t gStopping = 0;
-}
+namespace {
+volatile sig_atomic_t gStopping = 0;
+volatile sig_atomic_t gResetContent = 0;
+}   // namespace
 
-void signalHandler(int /* signal */)
-{
-    gStopping = 1;
-}
+void signalHandler(int /* signal */) { gStopping = 1; }
+
+void resetContentHandler(int /* signal */) { gResetContent = 1; }
 
 struct ShmManager
 {
-    ShmManager(uint64_t _shmId, const vector<string>& _segments, const vector<string>& _regions)
+    ShmManager(uint64_t _shmId, const vector<string>& _segments, const vector<string>& _regions, bool zero = true)
         : shmId(fair::mq::shmem::makeShmIdStr(_shmId))
+    {
+        LOG(info) << "Starting ShmManager for shmId: " << shmId;
+        LOG(info) << "Performing full reset...";
+        FullReset();
+        LOG(info) << "Done.";
+        LOG(info) << "Adding managed segments...";
+        AddSegments(_segments, zero);
+        LOG(info) << "Done.";
+        LOG(info) << "Adding unmanaged regions...";
+        AddRegions(_regions, zero);
+        LOG(info) << "Done.";
+        LOG(info) << "Shared memory is ready for use.";
+    }
+
+    void AddSegments(const vector<string>& _segments, bool zero)
     {
         for (const auto& s : _segments) {
-            vector<string> segmentConf;
-            boost::algorithm::split(segmentConf, s, boost::algorithm::is_any_of(","));
-            if (segmentConf.size() != 2) {
-                LOG(error) << "incorrect format for --segments. Expecting pairs of <id>,<size>.";
+            vector<string> conf;
+            boost::algorithm::split(conf, s, boost::algorithm::is_any_of(","));
+            if (conf.size() != 3) {
+                LOG(error) << "incorrect format for --segments. Expecting pairs of <id>,<size><numaid>.";
                 fair::mq::shmem::Monitor::Cleanup(fair::mq::shmem::ShmId{shmId});
-                throw runtime_error("incorrect format for --segments. Expecting pairs of <id>,<size>.");
+                throw runtime_error("incorrect format for --segments. Expecting pairs of <id>,<size>,<numaid>.");
             }
-            uint16_t id = stoi(segmentConf.at(0));
-            uint64_t size = stoull(segmentConf.at(1));
+            uint16_t id = stoi(conf.at(0));
+            uint64_t size = stoull(conf.at(1));
+            segmentCfgs.emplace_back(fair::mq::shmem::SegmentConfig{id, size, "rbtree_best_fit"});
+
             auto ret = segments.emplace(id, fair::mq::shmem::Segment(shmId, id, size, fair::mq::shmem::rbTreeBestFit));
             fair::mq::shmem::Segment& segment = ret.first->second;
-            LOG(info) << "Created segment " << id << " of size " << segment.GetSize() << ", starting at " << segment.GetData() << ". Locking...";
+            LOG(info) << "Created segment " << id << " of size " << segment.GetSize()
+                      << ", starting at " << segment.GetData() << ". Locking...";
             segment.Lock();
             LOG(info) << "Done.";
-            LOG(info) << "Zeroing...";
-            segment.Zero();
-            LOG(info) << "Done.";
-        }
-
-        for (const auto& r : _regions) {
-            vector<string> regionConf;
-            boost::algorithm::split(regionConf, r, boost::algorithm::is_any_of(","));
-            if (regionConf.size() != 2) {
-                LOG(error) << "incorrect format for --regions. Expecting pairs of <id>,<size>.";
-                fair::mq::shmem::Monitor::Cleanup(fair::mq::shmem::ShmId{shmId});
-                throw runtime_error("incorrect format for --regions. Expecting pairs of <id>,<size>.");
+            if (zero) {
+                LOG(info) << "Zeroing...";
+                segment.Zero();
+                LOG(info) << "Done.";
             }
-            uint16_t id = stoi(regionConf.at(0));
-            uint64_t size = stoull(regionConf.at(1));
+        }
+    }
+
+    void AddRegions(const vector<string>& _regions, bool zero)
+    {
+        for (const auto& r : _regions) {
+            vector<string> conf;
+            boost::algorithm::split(conf, r, boost::algorithm::is_any_of(","));
+            if (conf.size() != 3) {
+                LOG(error) << "incorrect format for --regions. Expecting pairs of <id>,<size>,<numaid>.";
+                fair::mq::shmem::Monitor::Cleanup(fair::mq::shmem::ShmId{shmId});
+                throw runtime_error("incorrect format for --regions. Expecting pairs of <id>,<size>,<numaid>.");
+            }
+            uint16_t id = stoi(conf.at(0));
+            uint64_t size = stoull(conf.at(1));
+            fair::mq::RegionConfig cfg;
+            cfg.id = id;
+            cfg.size = size;
+            regionCfgs.push_back(cfg);
+
             auto ret = regions.emplace(id, make_unique<fair::mq::shmem::UnmanagedRegion>(shmId, id, size));
             fair::mq::shmem::UnmanagedRegion& region = *(ret.first->second);
-            LOG(info) << "Created unamanged region " << id << " of size " << region.GetSize() << ", starting at " << region.GetData() << ". Locking...";
+            LOG(info) << "Created unamanged region " << id << " of size " << region.GetSize()
+                      << ", starting at " << region.GetData() << ". Locking...";
             region.Lock();
             LOG(info) << "Done.";
-            LOG(info) << "Zeroing...";
-            region.Zero();
-            LOG(info) << "Done.";
+            if (zero) {
+                LOG(info) << "Zeroing...";
+                region.Zero();
+                LOG(info) << "Done.";
+            }
         }
     }
 
+    bool CheckPresence()
+    {
+        std::lock_guard<std::mutex> lock(localMtx);
+        for (const auto& sc : segmentCfgs) {
+            if (!(fair::mq::shmem::Monitor::SegmentIsPresent(fair::mq::shmem::ShmId{shmId}, sc.id))) {
+                return false;
+            }
+        }
+        for (const auto& rc : regionCfgs) {
+            if (!(fair::mq::shmem::Monitor::RegionIsPresent(fair::mq::shmem::ShmId{shmId}, rc.id.value()))) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     void ResetContent()
     {
-        fair::mq::shmem::Monitor::ResetContent(fair::mq::shmem::ShmId{shmId});
+        std::lock_guard<std::mutex> lock(localMtx);
+        fair::mq::shmem::Monitor::ResetContent(fair::mq::shmem::ShmId{shmId}, segmentCfgs, regionCfgs);
+    }
+
+    void FullReset()
+    {
+        segments.clear();
+        regions.clear();
+        fair::mq::shmem::Monitor::Cleanup(fair::mq::shmem::ShmId{shmId});
     }
 
     ~ShmManager()
@@ -95,8 +148,11 @@ struct ShmManager
     }
 
     std::string shmId;
+    std::mutex localMtx;
     map<uint16_t, fair::mq::shmem::Segment> segments;
     map<uint16_t, unique_ptr<fair::mq::shmem::UnmanagedRegion>> regions;
+    std::vector<fair::mq::shmem::SegmentConfig> segmentCfgs;
+    std::vector<fair::mq::RegionConfig> regionCfgs;
 };
 
 int main(int argc, char** argv)
@@ -105,8 +161,11 @@ int main(int argc, char** argv)
 
     signal(SIGINT, signalHandler);
     signal(SIGTERM, signalHandler);
+    signal(SIGUSR1, resetContentHandler);
 
     try {
+        bool nozero = false;
+        bool checkPresence = true;
         uint64_t shmId = 0;
         vector<string> segments;
         vector<string> regions;
@@ -114,8 +173,10 @@ int main(int argc, char** argv)
         options_description desc("Options");
         desc.add_options()
             ("shmid", value<uint64_t>(&shmId)->required(), "Shm id")
-            ("segments", value<vector<string>>(&segments)->multitoken()->composing(), "Segments, as <id>,<size> <id>,<size> <id>,<size> ...")
-            ("regions",  value<vector<string>>(&regions)->multitoken()->composing(), "Regions, as <id>,<size> <id>,<size> <id>,<size> ...")
+            ("segments", value<vector<string>>(&segments)->multitoken()->composing(), "Segments, as <id>,<size>,<numaid> <id>,<size>,<numaid> <id>,<size>,<numaid> ... (numaid: -2 disabled, -1 interleave, >=0 node)")
+            ("regions", value<vector<string>>(&regions)->multitoken()->composing(), "Regions, as <id>,<size> <id>,<size>,<numaid> <id>,<size>,<numaid> ...")
+            ("nozero", value<bool>(&nozero)->default_value(false)->implicit_value(true), "Do not zero segments after initialization")
+            ("check-presence", value<bool>(&checkPresence)->default_value(true)->implicit_value(true), "Check periodically if configured segments/regions are still present, and cleanup and leave if they are not")
             ("help,h", "Print help");
 
         variables_map vm;
@@ -128,15 +189,35 @@ int main(int argc, char** argv)
 
         notify(vm);
 
-        ShmManager shmManager(shmId, segments, regions);
+        ShmManager shmManager(shmId, segments, regions, !nozero);
 
-        while (!gStopping) {
-            std::this_thread::sleep_for(std::chrono::milliseconds(50));
+        std::thread resetContentThread([&shmManager]() {
+            while (!gStopping) {
+                std::this_thread::sleep_for(std::chrono::milliseconds(50));
+                if (gResetContent == 1) {
+                    LOG(info) << "Resetting content for shmId " << shmManager.shmId;
+                    shmManager.ResetContent();
+                    gResetContent = 0;
+                    LOG(info) << "Done resetting content for shmId " << shmManager.shmId;
+                }
+            }
+        });
+
+        if (checkPresence) {
+            while (!gStopping) {
+                if (shmManager.CheckPresence() == false) {
+                    LOG(error) << "Failed to find segments, exiting.";
+                    gStopping = true;
+                }
+                std::this_thread::sleep_for(std::chrono::milliseconds(500));
+            }
         }
 
+        resetContentThread.join();
+
         LOG(info) << "stopping.";
     } catch (exception& e) {
-        LOG(error) << "Unhandled Exception reached the top of main: " << e.what() << ", application will now exit";
+        LOG(error) << "Exception reached the top of main: " << e.what() << ", exiting";
         return 2;
     }
 

examples/region/processor.cxx

@@ -0,0 +1,80 @@
+/********************************************************************************
+ * Copyright (C) 2014-2021 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
+ *                                                                              *
+ *              This software is distributed under the terms of the             *
+ *              GNU Lesser General Public Licence (LGPL) version 3,             *
+ *                  copied verbatim in the file "LICENSE"                       *
+ ********************************************************************************/
+
+#include <fairmq/Device.h>
+#include <fairmq/runDevice.h>
+#include <memory>
+
+namespace bpo = boost::program_options;
+using namespace std;
+using namespace fair::mq;
+
+namespace {
+
+struct Processor : Device
+{
+    void InitTask() override
+    {
+        fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
+        GetChannel("data1", 0).Transport()->SubscribeToRegionEvents([](RegionInfo info) {
+            LOG(info) << "Region event: " << info.event << ": "
+                      << (info.managed ? "managed" : "unmanaged") << ", id: " << info.id
+                      << ", ptr: " << info.ptr << ", size: " << info.size
+                      << ", flags: " << info.flags;
+        });
+    }
+
+    void Run() override
+    {
+        Channel& dataIn = GetChannel("data1", 0);
+        Channel& dataOut1 = GetChannel("data2", 0);
+        Channel& dataOut2 = GetChannel("data3", 0);
+
+        while (!NewStatePending()) {
+            fair::mq::Parts inParts;
+            dataIn.Receive(inParts);
+
+            fair::mq::Parts outParts1;
+            fair::mq::Parts outParts2;
+
+            for (const auto& inPart : inParts) {
+                outParts1.AddPart(NewMessage());
+                outParts1.fParts.back()->Copy(*inPart);
+
+                outParts2.AddPart(NewMessage());
+                outParts2.fParts.back()->Copy(*inPart);
+            }
+
+            dataOut1.Send(outParts1);
+            dataOut2.Send(outParts2);
+
+            if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
+                LOG(info) << "Configured max number of iterations reached. Leaving RUNNING state.";
+                break;
+            }
+        }
+    }
+
+    void ResetTask() override
+    {
+        GetChannel("data1", 0).Transport()->UnsubscribeFromRegionEvents();
+    }
+
+  private:
+    uint64_t fMaxIterations = 0;
+    uint64_t fNumIterations = 0;
+};
+
+}   // namespace
+
+void addCustomOptions(bpo::options_description& options)
+{
+    options.add_options()("max-iterations", bpo::value<uint64_t>()->default_value(0), "Maximum number of iterations of Run/ConditionalRun/OnData (0 - infinite)");
+}
+
+unique_ptr<Device> getDevice(ProgOptions& /*config*/) { return make_unique<Processor>(); }

examples/region/sampler.cxx

@@ -8,6 +8,7 @@
 
 #include <fairmq/Device.h>
 #include <fairmq/runDevice.h>
+#include <fairmq/tools/RateLimit.h>
 
 #include <cstdint>
 #include <mutex>
@@ -19,11 +20,15 @@ struct Sampler : fair::mq::Device
 {
     void InitTask() override
     {
+        fExternalRegion = fConfig->GetProperty<bool>("external-region");
         fMsgSize = fConfig->GetProperty<int>("msg-size");
         fLinger = fConfig->GetProperty<uint32_t>("region-linger");
         fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
+        fChanName = fConfig->GetProperty<std::string>("chan-name");
+        fSamplingRate = fConfig->GetProperty<float>("sampling-rate");
+        fRCSegmentSize = fConfig->GetProperty<uint64_t>("rc-segment-size");
 
-        GetChannel("data", 0).Transport()->SubscribeToRegionEvents([](fair::mq::RegionInfo info) {
+        GetChannel(fChanName, 0).Transport()->SubscribeToRegionEvents([](fair::mq::RegionInfo info) {
             LOG(info) << "Region event: " << info.event << ": "
                     << (info.managed ? "managed" : "unmanaged")
                     << ", id: " << info.id
@@ -34,76 +39,109 @@ struct Sampler : fair::mq::Device
 
         fair::mq::RegionConfig regionCfg;
         regionCfg.linger = fLinger; // delay in ms before region destruction to collect outstanding events
-        regionCfg.lock = true; // mlock region after creation
-        regionCfg.zero = true; // zero region content after creation
-        fRegion = fair::mq::UnmanagedRegionPtr(NewUnmanagedRegionFor("data", // region is created using the transport of this channel...
-                                                                0,      // ... and this sub-channel
-                                                                10000000, // region size
-                                                                [this](const std::vector<fair::mq::RegionBlock>& blocks) { // callback to be called when message buffers no longer needed by transport
-                                                                    std::lock_guard<std::mutex> lock(fMtx);
-                                                                    fNumUnackedMsgs -= blocks.size();
-                                                                    if (fMaxIterations > 0) {
-                                                                        LOG(info) << "Received " << blocks.size() << " acks";
-                                                                    }
-                                                                }, regionCfg));
+        // options for testing with an externally-created -region
+        if (fExternalRegion) {
+            regionCfg.id = 1;
+            regionCfg.removeOnDestruction = false;
+        }
+        regionCfg.lock = !fExternalRegion; // mlock region after creation
+        regionCfg.zero = !fExternalRegion; // zero region content after creation
+        regionCfg.rcSegmentSize = fRCSegmentSize; // size of the corresponding reference count segment
+        fRegion = fair::mq::UnmanagedRegionPtr(NewUnmanagedRegionFor(
+            fChanName, // region is created using the transport of this channel...
+            0,      // ... and this sub-channel
+            10000000, // region size
+            [this](const std::vector<fair::mq::RegionBlock>& blocks) { // callback to be called when message buffers no longer needed by transport
+                std::lock_guard<std::mutex> lock(fMtx);
+                fNumUnackedMsgs -= blocks.size();
+                if (fMaxIterations > 0) {
+                    LOG(info) << "Received " << blocks.size() << " acks";
+                }
+            },
+            regionCfg
+        ));
     }
 
-    bool ConditionalRun() override
+    void Run() override
     {
-        fair::mq::MessagePtr msg(NewMessageFor("data", // channel
-                                            0, // sub-channel
-                                            fRegion, // region
-                                            fRegion->GetData(), // ptr within region
-                                            fMsgSize, // offset from ptr
-                                            nullptr // hint
-                                            ));
 
-        // static_cast<char*>(fRegion->GetData())[3] = 97;
-        // LOG(info) << "check: " << static_cast<char*>(fRegion->GetData())[3];
-        // std::this_thread::sleep_for(std::chrono::seconds(1));
+        fair::mq::tools::RateLimiter rateLimiter(fSamplingRate);
 
-        std::lock_guard<std::mutex> lock(fMtx);
-        ++fNumUnackedMsgs;
-        if (Send(msg, "data", 0) > 0) {
-            if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
-                LOG(info) << "Configured maximum number of iterations reached. Leaving RUNNING state.";
-                return false;
+        while (!NewStatePending()) {
+            fair::mq::Parts parts;
+            // make 64 parts
+            for (int i = 0; i < 64; ++i) {
+                parts.AddPart(NewMessageFor(
+                    fChanName, // channel
+                    0, // sub-channel
+                    fRegion, // region
+                    fRegion->GetData(), // ptr within region
+                    fMsgSize, // offset from ptr
+                    nullptr // hint
+                    ));
+            }
+
+            std::lock_guard<std::mutex> lock(fMtx);
+            fNumUnackedMsgs += parts.Size();
+            if (Send(parts, fChanName, 0) > 0) {
+                if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
+                    LOG(info) << "Configured maximum number of iterations reached. Stopping sending.";
+                    break;
+                }
+                if (fSamplingRate > 0.001) {
+                    rateLimiter.maybe_sleep();
+                }
             }
         }
 
-        return true;
+        // wait for all acks to arrive
+        while (!NewStatePending()) {
+            {
+                std::lock_guard<std::mutex> lock(fMtx);
+                if (fNumUnackedMsgs == 0) {
+                    break;
+                }
+            }
+            std::this_thread::sleep_for(std::chrono::milliseconds(25));
+        }
+
+        if (fNumUnackedMsgs != 0) {
+            LOG(info) << "Done, still not acknowledged: " << fNumUnackedMsgs;
+        } else {
+            LOG(info) << "All acknowledgements received.";
+        }
     }
 
     void ResetTask() override
     {
         fRegion.reset();
-        {
-            std::lock_guard<std::mutex> lock(fMtx);
-            if (fNumUnackedMsgs != 0) {
-                LOG(info) << "Done, still not acknowledged: " << fNumUnackedMsgs;
-            } else {
-                LOG(info) << "All acknowledgements received.";
-            }
-        }
-        GetChannel("data", 0).Transport()->UnsubscribeFromRegionEvents();
+        GetChannel(fChanName, 0).Transport()->UnsubscribeFromRegionEvents();
     }
 
   private:
+    int fExternalRegion = false;
     int fMsgSize = 10000;
     uint32_t fLinger = 100;
     uint64_t fMaxIterations = 0;
     uint64_t fNumIterations = 0;
+    uint64_t fRCSegmentSize = 10000000;
     fair::mq::UnmanagedRegionPtr fRegion = nullptr;
     std::mutex fMtx;
     uint64_t fNumUnackedMsgs = 0;
+    std::string fChanName;
+    float fSamplingRate = 0.;
 };
 
 void addCustomOptions(bpo::options_description& options)
 {
     options.add_options()
+        ("chan-name", bpo::value<std::string>()->default_value("data"), "name of the output channel")
         ("msg-size", bpo::value<int>()->default_value(1000), "Message size in bytes")
+        ("sampling-rate", bpo::value<float>()->default_value(0.), "Sampling rate (Hz).")
         ("region-linger", bpo::value<uint32_t>()->default_value(100), "Linger period for regions")
-        ("max-iterations", bpo::value<uint64_t>()->default_value(0), "Maximum number of iterations of Run/ConditionalRun/OnData (0 - infinite)");
+        ("max-iterations", bpo::value<uint64_t>()->default_value(0), "Maximum number of iterations of Run/ConditionalRun/OnData (0 - infinite)")
+        ("external-region", bpo::value<bool>()->default_value(false), "Use region created by another process")
+        ("rc-segment-size", bpo::value<uint64_t>()->default_value(10000000), "Size of the reference count segment for Unamanged Region");
 }
 
 std::unique_ptr<fair::mq::Device> getDevice(fair::mq::ProgOptions& /*config*/)

examples/region/sink.cxx

@@ -22,7 +22,8 @@ struct Sink : Device
     {
         // Get the fMaxIterations value from the command line options (via fConfig)
         fMaxIterations = fConfig->GetProperty<uint64_t>("max-iterations");
-        GetChannel("data", 0).Transport()->SubscribeToRegionEvents([](RegionInfo info) {
+        fChanName = fConfig->GetProperty<std::string>("chan-name");
+        GetChannel(fChanName, 0).Transport()->SubscribeToRegionEvents([](RegionInfo info) {
             LOG(info) << "Region event: " << info.event << ": "
                       << (info.managed ? "managed" : "unmanaged") << ", id: " << info.id
                       << ", ptr: " << info.ptr << ", size: " << info.size
@@ -32,15 +33,11 @@ struct Sink : Device
 
     void Run() override
     {
-        Channel& dataInChannel = GetChannel("data", 0);
+        Channel& dataIn = GetChannel(fChanName, 0);
 
         while (!NewStatePending()) {
-            auto msg(dataInChannel.Transport()->CreateMessage());
-            dataInChannel.Receive(msg);
-
-            // void* ptr = msg->GetData();
-            // char* cptr = static_cast<char*>(ptr);
-            // LOG(info) << "check: " << cptr[3];
+            fair::mq::Parts parts;
+            dataIn.Receive(parts);
 
             if (fMaxIterations > 0 && ++fNumIterations >= fMaxIterations) {
                 LOG(info) << "Configured max number of iterations reached. Leaving RUNNING state.";
@@ -51,22 +48,22 @@ struct Sink : Device
 
     void ResetTask() override
     {
-        GetChannel("data", 0).Transport()->UnsubscribeFromRegionEvents();
+        GetChannel(fChanName, 0).Transport()->UnsubscribeFromRegionEvents();
     }
 
   private:
     uint64_t fMaxIterations = 0;
     uint64_t fNumIterations = 0;
+    std::string fChanName;
 };
 
 }   // namespace
 
 void addCustomOptions(bpo::options_description& options)
 {
-    options.add_options()(
-        "max-iterations",
-        bpo::value<uint64_t>()->default_value(0),
-        "Maximum number of iterations of Run/ConditionalRun/OnData (0 - infinite)");
+    options.add_options()
+        ("chan-name", bpo::value<std::string>()->default_value("data"), "name of the input channel")
+        ("max-iterations", bpo::value<uint64_t>()->default_value(0), "Maximum number of iterations of Run/ConditionalRun/OnData (0 - infinite)");
 }
 
 unique_ptr<Device> getDevice(ProgOptions& /*config*/) { return make_unique<Sink>(); }

examples/region/test-ex-region.sh.in

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
 
 transport="zeromq"
@@ -14,7 +16,7 @@ chan="data"
 chanAddr="/tmp/fmq_$session""_""$chan""_""$transport"
 
 # setup a trap to kill everything if the test fails/timeouts
-trap 'kill -TERM $SAMPLER_PID; kill -TERM $SINK_PID; wait $SAMPLER_PID; wait $SINK_PID; @CMAKE_BINARY_DIR@/fairmq/fairmq-shmmonitor --cleanup --session $SESSION; rm $chanAddr' TERM
+trap 'set +e; kill -TERM $SAMPLER_PID; kill -TERM $SINK_PID; wait $SAMPLER_PID; wait $SINK_PID; @CMAKE_BINARY_DIR@/fairmq/fairmq-shmmonitor --cleanup --session $SESSION; rm $chanAddr; exit 0' TERM
 
 SAMPLER="fairmq-ex-region-sampler"
 SAMPLER+=" --id sampler1"
@@ -22,6 +24,7 @@ SAMPLER+=" --transport $transport"
 SAMPLER+=" --severity debug"
 SAMPLER+=" --session $session"
 SAMPLER+=" --shm-segment-size 100000000"
+SAMPLER+=" --shm-monitor true"
 SAMPLER+=" --verbosity veryhigh"
 SAMPLER+=" --control static --color false"
 SAMPLER+=" --max-iterations 1"
@@ -37,6 +40,7 @@ SINK+=" --transport $transport"
 SINK+=" --severity debug"
 SINK+=" --session $session"
 SINK+=" --shm-segment-size 100000000"
+SINK+=" --shm-monitor true"
 SINK+=" --verbosity veryhigh"
 SINK+=" --control static --color false"
 SINK+=" --max-iterations 1"
@@ -48,4 +52,6 @@ SINK_PID=$!
 wait $SAMPLER_PID
 wait $SINK_PID
 
+set +e
 rm $chanAddr
+exit 0

examples/req-rep/CMakeLists.txt

@@ -1,52 +1,11 @@
- ################################################################################
- #    Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    #
- #                                                                              #
- #              This software is distributed under the terms of the             #
- #              GNU Lesser General Public Licence (LGPL) version 3,             #
- #                  copied verbatim in the file "LICENSE"                       #
- ################################################################################
+################################################################################
+# Copyright (C) 2014-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
+#                                                                              #
+#              This software is distributed under the terms of the             #
+#              GNU Lesser General Public Licence (LGPL) version 3,             #
+#                  copied verbatim in the file "LICENSE"                       #
+################################################################################
 
-add_executable(fairmq-ex-req-rep-client client.cxx)
-target_link_libraries(fairmq-ex-req-rep-client PRIVATE FairMQ)
-
-
-add_executable(fairmq-ex-req-rep-server server.cxx)
-target_link_libraries(fairmq-ex-req-rep-server PRIVATE FairMQ)
-
-add_custom_target(ExampleReqRep DEPENDS fairmq-ex-req-rep-client fairmq-ex-req-rep-server)
-
-set(EX_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_BINARY_DIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-req-rep.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-req-rep.sh)
-
-# test
-
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test-ex-req-rep.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh)
-
-add_test(NAME Example.ReqRep.zeromq COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh zeromq)
-set_tests_properties(Example.ReqRep.zeromq PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Received reply from server: ")
-
-add_test(NAME Example.ReqRep.shmem COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test-ex-req-rep.sh shmem)
-set_tests_properties(Example.ReqRep.shmem PROPERTIES TIMEOUT "30" PASS_REGULAR_EXPRESSION "Received reply from server: ")
-
-# install
-
-install(
-  TARGETS
-  fairmq-ex-req-rep-client
-  fairmq-ex-req-rep-server
-
-  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
-  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
-)
-
-# configure run script with different executable paths for build and for install directories
-set(EX_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR})
-set(FAIRMQ_BIN_DIR ${CMAKE_INSTALL_PREFIX}/${PROJECT_INSTALL_BINDIR}/fairmq)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fairmq-start-ex-req-rep.sh.in ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-req-rep.sh_install)
-
-install(
-  PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/fairmq-start-ex-req-rep.sh_install
-  DESTINATION ${PROJECT_INSTALL_BINDIR}
-  RENAME fairmq-start-ex-req-rep.sh
+add_example(NAME req-rep
+  DEVICE client server
 )

examples/req-rep/test-ex-req-rep.sh.in

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 export FAIRMQ_PATH=@FAIRMQ_BIN_DIR@
 
 transport="zeromq"
@@ -13,7 +15,7 @@ chan="data"
 chanAddr="/tmp/fmq_$session""_""$chan""_""$transport"
 
 # setup a trap to kill everything if the test fails/timeouts
-trap 'kill -TERM $CLIENT_PID; kill -TERM $SERVER_PID; wait $CLIENT_PID; wait $SERVER_PID; rm $chanAddr' TERM
+trap 'set +e; kill -TERM $CLIENT_PID; kill -TERM $SERVER_PID; wait $CLIENT_PID; wait $SERVER_PID; rm $chanAddr; exit 0' TERM
 
 CLIENT="fairmq-ex-req-rep-client"
 CLIENT+=" --id client"
@@ -21,6 +23,7 @@ CLIENT+=" --transport $transport"
 CLIENT+=" --verbosity veryhigh"
 CLIENT+=" --session $session"
 CLIENT+=" --shm-segment-size 100000000"
+CLIENT+=" --shm-monitor true"
 CLIENT+=" --control static --color false"
 CLIENT+=" --max-iterations 1"
 CLIENT+=" --channel-config name=$chan,type=req,method=connect,rateLogging=0,address=ipc://$chanAddr"
@@ -33,6 +36,7 @@ SERVER+=" --transport $transport"
 SERVER+=" --verbosity veryhigh"
 SERVER+=" --session $session"
 SERVER+=" --shm-segment-size 100000000"
+SERVER+=" --shm-monitor true"
 SERVER+=" --control static --color false"
 SERVER+=" --max-iterations 1"
 SERVER+=" --channel-config name=$chan,type=rep,method=bind,rateLogging=0,address=ipc://$chanAddr"
@@ -43,4 +47,6 @@ SERVER_PID=$!
 wait $CLIENT_PID
 wait $SERVER_PID
 
+set +e
 rm $chanAddr
+exit 0

fairmq/CMakeLists.txt

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (C) 2012-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
+# Copyright (C) 2012-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
 #                                                                              #
 #              This software is distributed under the terms of the             #
 #              GNU Lesser General Public Licence (LGPL) version 3,             #
@@ -71,6 +71,7 @@ if(BUILD_FAIRMQ)
     shmem/Monitor.h
     shmem/Segment.h
     shmem/UnmanagedRegion.h
+    tools/Compiler.h
     tools/CppSTL.h
     tools/Exceptions.h
     tools/IO.h
@@ -107,18 +108,9 @@ if(BUILD_FAIRMQ)
     zeromq/UnmanagedRegion.h
     zeromq/Socket.h
     zeromq/TransportFactory.h
+    zeromq/ZMsg.h
   )
 
-  if(BUILD_OFI_TRANSPORT)
-    set(FAIRMQ_PRIVATE_HEADER_FILES ${FAIRMQ_PRIVATE_HEADER_FILES}
-      ofi/Context.h
-      ofi/ControlMessages.h
-      ofi/Message.h
-      ofi/Socket.h
-      ofi/TransportFactory.h
-    )
-  endif()
-
   ##########################
   # libFairMQ source files #
   ##########################
@@ -126,7 +118,6 @@ if(BUILD_FAIRMQ)
     Channel.cxx
     Device.cxx
     DeviceRunner.cxx
-    Error.cxx
     JSONParser.cxx
     MemoryResources.cxx
     Plugin.cxx
@@ -149,14 +140,6 @@ if(BUILD_FAIRMQ)
     tools/Unique.cxx
   )
 
-  if(BUILD_OFI_TRANSPORT)
-    set(FAIRMQ_SOURCE_FILES ${FAIRMQ_SOURCE_FILES}
-      ofi/Context.cxx
-      ofi/Message.cxx
-      ofi/Socket.cxx
-    )
-  endif()
-
 
   ###################
   # configure files #
@@ -180,14 +163,26 @@ if(BUILD_FAIRMQ)
   ############################
   # preprocessor definitions #
   ############################
-  target_compile_definitions(${target} PUBLIC BOOST_ERROR_CODE_HEADER_ONLY)
+  target_compile_definitions(${target} PUBLIC
+    BOOST_ERROR_CODE_HEADER_ONLY
+    BOOST_ASIO_HAS_HAS_STD_CHRONO
+  )
   if(FAIRMQ_DEBUG_MODE)
     target_compile_definitions(${target} PUBLIC FAIRMQ_DEBUG_MODE)
   endif()
-  if(BUILD_OFI_TRANSPORT)
-    target_compile_definitions(${target} PRIVATE BUILD_OFI_TRANSPORT)
+  target_compile_definitions(${target} PUBLIC
+    FAIRMQ_HAS_STD_FILESYSTEM=${FAIRMQ_HAS_STD_FILESYSTEM}
+    FAIRMQ_HAS_STD_PMR=${FAIRMQ_HAS_STD_PMR}
+  )
+  if(DEFINED FAIRMQ_CHANNEL_DEFAULT_AUTOBIND)
+    # translate CMake boolean (TRUE, FALSE, 0, 1, OFF, ON) into C++ boolean literal (true, false)
+    if(FAIRMQ_CHANNEL_DEFAULT_AUTOBIND)
+      set(value "true")
+    else()
+      set(value "false")
+    endif()
+    target_compile_definitions(${target} PUBLIC FAIRMQ_CHANNEL_DEFAULT_AUTOBIND=${value})
   endif()
-  target_compile_definitions(${target} PUBLIC FAIRMQ_HAS_STD_FILESYSTEM=${FAIRMQ_HAS_STD_FILESYSTEM})
 
 
   #######################
@@ -205,13 +200,6 @@ if(BUILD_FAIRMQ)
   ##################
   # link libraries #
   ##################
-  if(BUILD_OFI_TRANSPORT)
-    set(OFI_DEPS
-      asio::asio
-      asiofi::asiofi
-    )
-  endif()
-
   target_link_libraries(${target}
     INTERFACE # only consumers link against interface dependencies
     Boost::container
@@ -222,14 +210,13 @@ if(BUILD_FAIRMQ)
     $<$<PLATFORM_ID:Linux>:rt>
     Boost::boost
     Boost::program_options
-    Boost::filesystem
+    Boost::filesystem # still needed for Boost.DLL
     Boost::regex
     FairLogger::FairLogger
 
     PRIVATE # only libFairMQ links against private dependencies
     libzmq
     PicoSHA2
-    ${OFI_DEPS}
   )
   set_target_properties(${target} PROPERTIES
     VERSION ${PROJECT_VERSION}
@@ -343,10 +330,3 @@ if(BUILD_FAIRMQ)
     )
   endforeach()
 endif()
-
-####################
-# external plugins #
-####################
-if(BUILD_PMIX_PLUGIN)
-  add_subdirectory(plugins/PMIx)
-endif()

fairmq/Channel.h

@@ -19,6 +19,7 @@
 
 #include <cstdint>   // int64_t
 #include <memory>   // unique_ptr, shared_ptr
+#include <ostream>
 #include <stdexcept>
 #include <string>
 #include <utility>   // std::move
@@ -378,7 +379,19 @@ class Channel
     static constexpr int DefaultRateLogging = 1;
     static constexpr int DefaultPortRangeMin = 22000;
     static constexpr int DefaultPortRangeMax = 23000;
+#ifdef FAIRMQ_CHANNEL_DEFAULT_AUTOBIND
+    static constexpr bool DefaultAutoBind = FAIRMQ_CHANNEL_DEFAULT_AUTOBIND;
+#else
     static constexpr bool DefaultAutoBind = true;
+#endif
+
+    friend std::ostream& operator<<(std::ostream& os, const Channel& ch)
+    {
+        return os << "name: " << ch.fName
+                  << ", type: "  << ch.fType
+                  << ", method: "  << ch.fMethod
+                  << ", address: "  << ch.fAddress;
+    }
 
   private:
     std::shared_ptr<TransportFactory> fTransportFactory;
@@ -416,10 +429,10 @@ class Channel
                     msg.get()
                 ));
                 msg.release();
-                msg = move(msgWrapper);
+                msg = std::move(msgWrapper);
             } else {
                 MessagePtr newMsg(NewMessage());
-                msg = move(newMsg);
+                msg = std::move(newMsg);
             }
         }
     }
@@ -437,10 +450,10 @@ class Channel
                         msg.get()
                     ));
                     msg.release();
-                    msg = move(msgWrapper);
+                    msg = std::move(msgWrapper);
                 } else {
                     MessagePtr newMsg(NewMessage());
-                    msg = move(newMsg);
+                    msg = std::move(newMsg);
                 }
             }
         }
@@ -450,7 +463,7 @@ class Channel
     {
         if (fTransportType != msg->GetType()) {
             MessagePtr newMsg(NewMessage());
-            msg = move(newMsg);
+            msg = std::move(newMsg);
         }
     }
 
@@ -461,7 +474,7 @@ class Channel
             if (fTransportType != msg->GetType()) {
 
                 MessagePtr newMsg(NewMessage());
-                msg = move(newMsg);
+                msg = std::move(newMsg);
             }
         }
     }

fairmq/Device.cxx

@@ -1,19 +1,24 @@
 /********************************************************************************
- * Copyright (C) 2012-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
+ * Copyright (C) 2012-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
  *                  copied verbatim in the file "LICENSE"                       *
  ********************************************************************************/
 
-#include <algorithm>                    // std::max, std::any_of
-#include <boost/algorithm/string.hpp>   // join/split
-#include <chrono>
+// FairMQ
 #include <fairmq/Device.h>
 #include <fairmq/Tools.h>
+
+// boost
+#include <boost/algorithm/string.hpp>   // join/split
+
+// std
+#include <algorithm>   // std::max, std::any_of
+#include <chrono>
 #include <iomanip>
 #include <list>
-#include <memory>                       // std::make_unique
+#include <memory>   // std::make_unique
 #include <mutex>
 #include <thread>
 
@@ -72,6 +77,9 @@ Device::Device(ProgOptions& config, tools::Version version)
     : Device(&config, version)
 {}
 
+/// TODO: Remove this once Device::fChannels is no longer public
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 Device::Device(ProgOptions* config, tools::Version version)
     : fTransportFactory(nullptr)
     , fInternalConfig(config ? nullptr : make_unique<ProgOptions>())
@@ -83,18 +91,15 @@ Device::Device(ProgOptions* config, tools::Version version)
     , fVersion(version)
     , fRate(DefaultRate)
     , fInitializationTimeoutInS(DefaultInitTimeout)
-    , fTransitioning(false)
 {
     SubscribeToNewTransition("device", [&](Transition transition) {
         LOG(trace) << "device notified on new transition: " << transition;
+        InterruptTransports();
+    });
 
-        switch (transition) {
-            case Transition::Stop:
-                UnblockTransports();
-                break;
-            default:
-                break;
-        }
+    fStateMachine.PrepareState([&](State state) {
+        LOG(trace) << "Resuming transports for " << state << " state";
+        ResumeTransports();
     });
 
     fStateMachine.HandleStates([&](State state) {
@@ -135,73 +140,7 @@ Device::Device(ProgOptions* config, tools::Version version)
 
     fStateMachine.Start();
 }
-
-void Device::TransitionTo(State s)
-{
-    {
-        lock_guard<mutex> lock(fTransitionMtx);
-        if (fTransitioning) {
-            LOG(debug) << "Attempting a transition with TransitionTo() while another one is already in progress";
-            throw OngoingTransition("Attempting a transition with TransitionTo() while another one is already in progress");
-        }
-        fTransitioning = true;
-    }
-
-    using mq::State;
-
-    StateQueue sq;
-    StateSubscription ss(tools::ToString(fId, ".TransitionTo"), fStateMachine, sq);
-
-    State currentState = GetCurrentState();
-
-    while (s != currentState) {
-        switch (currentState) {
-            case State::Idle:
-                if (s == State::Exiting) { ChangeState(Transition::End); }
-                else { ChangeState(Transition::InitDevice); }
-            break;
-            case State::InitializingDevice:
-                ChangeState(Transition::CompleteInit);
-            break;
-            case State::Initialized:
-                if (s == State::Exiting || s == State::Idle) { ChangeState(Transition::ResetDevice); }
-                else { ChangeState(Transition::Bind); }
-            break;
-            case State::Bound:
-                if (s == State::DeviceReady || s == State::Ready || s == State::Running) { ChangeState(Transition::Connect); }
-                else { ChangeState(Transition::ResetDevice); }
-            break;
-            case State::DeviceReady:
-                if (s == State::Running || s == State::Ready) { ChangeState(Transition::InitTask); }
-                else { ChangeState(Transition::ResetDevice); }
-            break;
-            case State::Ready:
-                if (s == State::Running) { ChangeState(Transition::Run); }
-                else { ChangeState(Transition::ResetTask); }
-            break;
-            case State::Running:
-                ChangeState(Transition::Stop);
-            break;
-            case State::Binding:
-            case State::Connecting:
-            case State::InitializingTask:
-            case State::ResettingDevice:
-            case State::ResettingTask:
-                LOG(debug) << "TransitionTo ignoring state: " << currentState << " (expected, automatic transition).";
-            break;
-            default:
-                LOG(debug) << "TransitionTo ignoring state: " << currentState;
-            break;
-        }
-
-        currentState = sq.WaitForNext();
-    }
-
-    {
-        lock_guard<mutex> lock(fTransitionMtx);
-        fTransitioning = false;
-    }
-}
+#pragma GCC diagnostic pop
 
 void Device::InitWrapper()
 {
@@ -226,7 +165,7 @@ void Device::InitWrapper()
     unordered_map<string, int> infos = fConfig->GetChannelInfo();
     for (const auto& info : infos) {
         for (int i = 0; i < info.second; ++i) {
-            fChannels[info.first].emplace_back(info.first, i, fConfig->GetPropertiesStartingWith(tools::ToString("chans.", info.first, ".", i, ".")));
+            GetChannels()[info.first].emplace_back(info.first, i, fConfig->GetPropertiesStartingWith(tools::ToString("chans.", info.first, ".", i, ".")));
         }
     }
 
@@ -236,8 +175,7 @@ void Device::InitWrapper()
     string networkInterface = fConfig->GetProperty<string>("network-interface", DefaultNetworkInterface);
 
     // Fill the uninitialized channel containers
-    for (auto& channel : fChannels) {
-        int subChannelIndex = 0;
+    for (auto& channel : GetChannels()) {
         for (auto& subChannel : channel.second) {
             // set channel transport
             LOG(debug) << "Initializing transport for channel " << subChannel.fName << ": " << TransportNames.at(subChannel.fTransportType);
@@ -269,12 +207,10 @@ void Device::InitWrapper()
                 LOG(error) << "Cannot update configuration. Socket method (bind/connect) for channel '" << subChannel.fName << "' not specified.";
                 throw runtime_error(tools::ToString("Cannot update configuration. Socket method (bind/connect) for channel ", subChannel.fName, " not specified."));
             }
-
-            subChannelIndex++;
         }
     }
 
-    // ChangeState(Transition::Auto);
+    // ChangeStateOrThrow(Transition::Auto);
 }
 
 void Device::BindWrapper()
@@ -291,7 +227,7 @@ void Device::BindWrapper()
     Bind();
 
     if (!NewStatePending()) {
-        ChangeState(Transition::Auto);
+        ChangeStateOrThrow(Transition::Auto);
     }
 }
 
@@ -304,7 +240,7 @@ void Device::ConnectWrapper()
     // first attempt
     AttachChannels(fUninitializedConnectingChannels);
     // if not all channels could be connected, update their address values from config and retry
-    while (!fUninitializedConnectingChannels.empty()) {
+    while (!fUninitializedConnectingChannels.empty() && !NewStatePending()) {
         this_thread::sleep_for(chrono::milliseconds(sleepTimeInMS));
 
         for (auto& chan : fUninitializedConnectingChannels) {
@@ -317,20 +253,24 @@ void Device::ConnectWrapper()
 
         if (numAttempts++ > maxAttempts) {
             LOG(error) << "could not connect all channels after " << fInitializationTimeoutInS << " attempts";
+            LOG(error) << "following channels are still invalid:";
+            for (auto& chan : fUninitializedConnectingChannels) {
+                LOG(error) << "channel: " << *chan;
+            }
             throw runtime_error(tools::ToString("could not connect all channels after ", fInitializationTimeoutInS, " attempts"));
         }
 
         AttachChannels(fUninitializedConnectingChannels);
     }
 
-    if (fChannels.empty()) {
+    if (GetChannels().empty()) {
         LOG(warn) << "No channels created after finishing initialization";
     }
 
     Connect();
 
     if (!NewStatePending()) {
-        ChangeState(Transition::Auto);
+        ChangeStateOrThrow(Transition::Auto);
     }
 }
 
@@ -345,7 +285,7 @@ void Device::AttachChannels(vector<Channel*>& chans)
                 // remove the channel from the uninitialized container
                 itr = chans.erase(itr);
             } else {
-                LOG(error) << "failed to attach channel " << (*itr)->fName << " (" << (*itr)->fMethod << ")";
+                LOG(error) << "failed to attach channel " << (*itr)->fName << " (" << (*itr)->fMethod << " on " << (*itr)->fAddress << ")";
                 ++itr;
             }
         } else {
@@ -432,7 +372,7 @@ void Device::InitTaskWrapper()
     InitTask();
 
     if (!NewStatePending()) {
-        ChangeState(Transition::Auto);
+        ChangeStateOrThrow(Transition::Auto);
     }
 }
 
@@ -442,7 +382,7 @@ void Device::RunWrapper()
 
     unique_ptr<thread> rateLogger;
     // Check if rate logging thread is needed
-    const bool rateLogging = any_of(fChannels.cbegin(), fChannels.cend(), [](auto ch) {
+    const bool rateLogging = any_of(GetChannels().cbegin(), GetChannels().cend(), [](auto ch) {
         return any_of(ch.second.cbegin(), ch.second.cend(), [](auto sub) { return sub.fRateLogging > 0; });
     });
 
@@ -453,15 +393,9 @@ void Device::RunWrapper()
         if (rateLogging && rateLogger->joinable()) { rateLogger->join(); }
     });
 
-
-    // notify transports to resume transfers
-    for (auto& t : fTransports) {
-        t.second->Resume();
-    }
-
     // change to Error state in case of an exception, to release LogSocketRates
     tools::CallOnDestruction cod([&](){
-        ChangeState(Transition::ErrorFound);
+        ChangeStateOrThrow(Transition::ErrorFound);
     });
 
     PreRun();
@@ -469,7 +403,7 @@ void Device::RunWrapper()
     // process either data callbacks or ConditionalRun/Run
     if (fDataCallbacks) {
         // if only one input channel, do lightweight handling without additional polling.
-        if (fInputChannelKeys.size() == 1 && fChannels.at(fInputChannelKeys.at(0)).size() == 1) {
+        if (fInputChannelKeys.size() == 1 && GetChannels().at(fInputChannelKeys.at(0)).size() == 1) {
             HandleSingleChannelInput();
         } else {// otherwise do full handling with polling
             HandleMultipleChannelInput();
@@ -488,8 +422,7 @@ void Device::RunWrapper()
 
     // if Run() exited and the state is still RUNNING, transition to READY.
     if (!NewStatePending()) {
-        UnblockTransports();
-        ChangeState(Transition::Stop);
+        ChangeStateOrThrow(Transition::Stop);
     }
 
     PostRun();
@@ -517,7 +450,7 @@ void Device::HandleMultipleChannelInput()
     // check if more than one transport is used
     fMultitransportInputs.clear();
     for (const auto& k : fInputChannelKeys) {
-        mq::Transport t = fChannels.at(k).at(0).fTransportType;
+        mq::Transport t = GetChannel(k, 0).fTransportType;
         if (fMultitransportInputs.find(t) == fMultitransportInputs.end()) {
             fMultitransportInputs.insert(pair<mq::Transport, vector<string>>(t, vector<string>()));
             fMultitransportInputs.at(t).push_back(k);
@@ -527,13 +460,13 @@ void Device::HandleMultipleChannelInput()
     }
 
     for (const auto& mi : fMsgInputs) {
-        for (auto& i : fChannels.at(mi.first)) {
+        for (auto& i : GetChannels().at(mi.first)) {
             i.fMultipart = false;
         }
     }
 
     for (const auto& mi : fMultipartInputs) {
-        for (auto& i : fChannels.at(mi.first)) {
+        for (auto& i : GetChannels().at(mi.first)) {
             i.fMultipart = true;
         }
     }
@@ -544,16 +477,16 @@ void Device::HandleMultipleChannelInput()
     } else { // otherwise poll directly
         bool proceed = true;
 
-        PollerPtr poller(fChannels.at(fInputChannelKeys.at(0)).at(0).fTransportFactory->CreatePoller(fChannels, fInputChannelKeys));
+        PollerPtr poller(GetChannel(fInputChannelKeys.at(0), 0).fTransportFactory->CreatePoller(GetChannels(), fInputChannelKeys));
 
         while (!NewStatePending() && proceed) {
             poller->Poll(200);
 
             // check which inputs are ready and call their data handlers if they are.
             for (const auto& ch : fInputChannelKeys) {
-                for (unsigned int i = 0; i < fChannels.at(ch).size(); ++i) {
+                for (unsigned int i = 0; i < GetChannels().at(ch).size(); ++i) {
                     if (poller->CheckInput(ch, i)) {
-                        if (fChannels.at(ch).at(i).fMultipart) {
+                        if (GetChannel(ch, i).fMultipart) {
                             proceed = HandleMultipartInput(ch, fMultipartInputs.at(ch), i);
                         } else {
                             proceed = HandleMsgInput(ch, fMsgInputs.at(ch), i);
@@ -590,13 +523,13 @@ void Device::HandleMultipleTransportInput()
 void Device::PollForTransport(const TransportFactory* factory, const vector<string>& channelKeys)
 {
     try {
-        PollerPtr poller(factory->CreatePoller(fChannels, channelKeys));
+        PollerPtr poller(factory->CreatePoller(GetChannels(), channelKeys));
 
         while (!NewStatePending() && fMultitransportProceed) {
             poller->Poll(500);
 
             for (const auto& ch : channelKeys) {
-                for (unsigned int i = 0; i < fChannels.at(ch).size(); ++i) {
+                for (unsigned int i = 0; i < GetChannels().at(ch).size(); ++i) {
                     if (poller->CheckInput(ch, i)) {
                         lock_guard<mutex> lock(fMultitransportMutex);
 
@@ -604,7 +537,7 @@ void Device::PollForTransport(const TransportFactory* factory, const vector<stri
                             break;
                         }
 
-                        if (fChannels.at(ch).at(i).fMultipart) {
+                        if (GetChannel(ch, i).fMultipart) {
                             fMultitransportProceed = HandleMultipartInput(ch, fMultipartInputs.at(ch), i);
                         } else {
                             fMultitransportProceed = HandleMsgInput(ch, fMsgInputs.at(ch), i);
@@ -628,7 +561,7 @@ void Device::PollForTransport(const TransportFactory* factory, const vector<stri
 
 bool Device::HandleMsgInput(const string& chName, const InputMsgCallback& callback, int i)
 {
-    unique_ptr<Message> input(fChannels.at(chName).at(i).fTransportFactory->CreateMessage());
+    unique_ptr<Message> input(GetChannel(chName, i).fTransportFactory->CreateMessage());
 
     if (Receive(input, chName, i) >= 0) {
         return callback(input, i);
@@ -650,6 +583,8 @@ bool Device::HandleMultipartInput(const string& chName, const InputMultipartCall
 
 shared_ptr<TransportFactory> Device::AddTransport(mq::Transport transport)
 {
+    lock_guard<mutex> lock(fTransportMtx);
+
     if (transport == mq::Transport::DEFAULT) {
         transport = fDefaultTransportType;
     }
@@ -683,7 +618,7 @@ void Device::LogSocketRates()
     size_t chanNameLen = 0;
 
     // iterate over the channels map
-    for (auto& channel : fChannels) {
+    for (auto& channel : GetChannels()) {
         // iterate over the channels vector
         for (auto& subChannel : channel.second) {
             if (subChannel.fRateLogging > 0) {
@@ -767,10 +702,19 @@ void Device::LogSocketRates()
     }
 }
 
-void Device::UnblockTransports()
+void Device::InterruptTransports()
 {
-    for (auto& transport : fTransports) {
-        transport.second->Interrupt();
+    lock_guard<mutex> lock(fTransportMtx);
+    for (auto& [transportType, transport] : fTransports) {
+        transport->Interrupt();
+    }
+}
+
+void Device::ResumeTransports()
+{
+    lock_guard<mutex> lock(fTransportMtx);
+    for (auto& [transportType, transport] : fTransports) {
+        transport->Resume();
     }
 }
 
@@ -779,31 +723,38 @@ void Device::ResetTaskWrapper()
     ResetTask();
 
     if (!NewStatePending()) {
-        ChangeState(Transition::Auto);
+        ChangeStateOrThrow(Transition::Auto);
     }
 }
 
 void Device::ResetWrapper()
 {
-    for (auto& transport : fTransports) {
-        transport.second->Reset();
+    {
+        lock_guard<mutex> lock(fTransportMtx);
+        for (auto& [transportType, transport] : fTransports) {
+            transport->Reset();
+        }
+        fTransports.clear();
     }
 
     Reset();
 
-    fChannels.clear();
-    fTransports.clear();
+    GetChannels().clear();
     fTransportFactory.reset();
     if (!NewStatePending()) {
-        ChangeState(Transition::Auto);
+        ChangeStateOrThrow(Transition::Auto);
     }
 }
 
+/// TODO: Remove this once Device::fChannels is no longer public
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 Device::~Device()
 {
     UnsubscribeFromNewTransition("device");
     fStateMachine.StopHandlingStates();
     LOG(debug) << "Shutting down device " << fId;
 }
+#pragma GCC diagnostic pop
 
 } // namespace fair::mq

fairmq/Device.h

@@ -1,5 +1,5 @@
 /********************************************************************************
- * Copyright (C) 2021-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
+ * Copyright (C) 2021-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -9,12 +9,9 @@
 #ifndef FAIR_MQ_DEVICE_H
 #define FAIR_MQ_DEVICE_H
 
-#include <algorithm>   // find
-#include <atomic>
-#include <chrono>
-#include <cstddef>
-#include <fairlogger/Logger.h>
+// FairMQ
 #include <fairmq/Channel.h>
+#include <fairmq/Error.h>
 #include <fairmq/Message.h>
 #include <fairmq/Parts.h>
 #include <fairmq/ProgOptions.h>
@@ -24,6 +21,15 @@
 #include <fairmq/TransportFactory.h>
 #include <fairmq/Transports.h>
 #include <fairmq/UnmanagedRegion.h>
+
+// logger
+#include <fairlogger/Logger.h>
+
+// std
+#include <algorithm>   // find
+#include <atomic>
+#include <chrono>
+#include <cstddef>
 #include <functional>
 #include <memory>   // unique_ptr
 #include <mutex>
@@ -228,7 +234,7 @@ class Device
             }
         }
 
-        return GetChannel(chans.at(0), 0).Transport()->CreatePoller(fChannels, chans);
+        return GetChannel(chans.at(0), 0).Transport()->CreatePoller(GetChannels(), chans);
     }
 
     PollerPtr NewPoller(const std::vector<Channel*>& channels)
@@ -316,7 +322,7 @@ class Device
 
     Channel& GetChannel(const std::string& channelName, const int index = 0)
     try {
-        return fChannels.at(channelName).at(index);
+        return GetChannels().at(channelName).at(index);
     } catch (const std::out_of_range& oor) {
         LOG(error) << "GetChannel(): '" << channelName << "[" << index << "]' does not exist.";
         throw;
@@ -324,7 +330,7 @@ class Device
 
     size_t GetNumSubChannels(const std::string& channelName)
     try {
-        return fChannels.at(channelName).size();
+        return GetChannels().at(channelName).size();
     } catch (const std::out_of_range& oor) {
         LOG(error) << "GetNumSubChannels(): '" << channelName << "' does not exist.";
         throw;
@@ -335,7 +341,7 @@ class Device
     /// @param index sub-channel
     unsigned long GetNumberOfConnectedPeers(const std::string& channelName, int index = 0)
     {
-        return fChannels.at(channelName).at(index).GetNumberOfConnectedPeers();
+        return GetChannel(channelName, index).GetNumberOfConnectedPeers();
     }
 
     virtual void RegisterChannelEndpoints() {}
@@ -433,7 +439,23 @@ class Device
         fTransports;   ///< Container for transports
 
   public:
+    [[deprecated("Use GetChannels() instead.")]]
     std::unordered_map<std::string, std::vector<Channel>> fChannels;   ///< Device channels
+    std::unordered_map<std::string, std::vector<Channel>>& GetChannels()
+    {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+        return fChannels;
+#pragma GCC diagnostic pop
+    }
+    std::unordered_map<std::string, std::vector<Channel>> const& GetChannels() const
+    {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+        return fChannels;
+#pragma GCC diagnostic pop
+    }
+
     std::unique_ptr<ProgOptions> fInternalConfig;   ///< Internal program options configuration
     ProgOptions* fConfig;                           ///< Pointer to config (internal or external)
 
@@ -477,21 +499,49 @@ class Device
   public:
     /// @brief Request a device state transition
     /// @param transition state transition
+    /// @return whether the transition was successfully scheduled
     ///
     /// The state transition may not happen immediately, but when the current state evaluates the
     /// pending transition event and terminates. In other words, the device states are scheduled
     /// cooperatively.
-    bool ChangeState(const Transition transition) { return fStateMachine.ChangeState(transition); }
+    [[nodiscard]] bool ChangeState(const Transition transition)
+    {
+        return fStateMachine.ChangeState(transition);
+    }
     /// @brief Request a device state transition
     /// @param transition state transition
+    /// @return whether the transition was successfully scheduled
     ///
     /// The state transition may not happen immediately, but when the current state evaluates the
     /// pending transition event and terminates. In other words, the device states are scheduled
     /// cooperatively.
-    bool ChangeState(const std::string& transition)
+    [[nodiscard]] bool ChangeState(const std::string& transition)
     {
         return fStateMachine.ChangeState(GetTransition(transition));
     }
+    /// @brief Request a device state transition
+    /// @param transition state transition
+    /// @throws when the transition could not have been scheduled
+    ///
+    /// Throwing version of Device::ChangeState().
+    void ChangeStateOrThrow(Transition transition)
+    {
+        if(!ChangeState(transition)) {
+            auto const err = MakeErrorCode(ErrorCode::DeviceChangeStateFailed);
+            throw std::system_error(err.value(),
+                                    err.category(),
+                                    tools::ToString("Invalid transition: ", transition));
+        }
+    }
+    /// @brief Request a device state transition
+    /// @param transition state transition
+    /// @throws when the transition could not have been scheduled
+    ///
+    /// Throwing version of Device::ChangeState().
+    void ChangeStateOrThrow(std::string const& transition)
+    {
+        ChangeStateOrThrow(GetTransition(transition));
+    }
 
     /// @brief waits for the next state (any) to occur
     State WaitForNextState() { return fStateQueue.WaitForNext(); }
@@ -502,8 +552,6 @@ class Device
     /// @param state state to wait for
     void WaitForState(const std::string& state) { WaitForState(GetState(state)); }
 
-    void TransitionTo(State state);
-
     /// @brief Subscribe with a callback to state changes
     /// @param key id to identify your subscription
     /// @param callback callback (called with the new state as the parameter)
@@ -588,7 +636,10 @@ class Device
     void ResetWrapper();
 
     /// Notifies transports to cease any blocking activity
-    void UnblockTransports();
+    void InterruptTransports();
+
+    /// Notifies transports to resume any blocking activity
+    void ResumeTransports();
 
     /// Shuts down the transports and the device
     void Exit() {}
@@ -622,15 +673,12 @@ class Device
 
     const tools::Version fVersion;
     float fRate;                  ///< Rate limiting for ConditionalRun
-    uint64_t fMaxRunRuntimeInS;   ///< Maximum runtime for the Running state handler, after which
-                                  ///< state will change to Ready (in seconds, 0 for no limit).
     int fInitializationTimeoutInS;
     std::vector<std::string> fRawCmdLineArgs;
 
     StateQueue fStateQueue;
 
-    std::mutex fTransitionMtx;
-    bool fTransitioning;
+    std::mutex fTransportMtx;   ///< guards access to transports container
 };
 
 }   // namespace fair::mq

fairmq/DeviceRunner.cxx

@@ -152,7 +152,7 @@ auto DeviceRunner::Run() -> int
     fDevice->RegisterChannelEndpoints();
     if (fConfig.Count("print-channels")) {
         fDevice->PrintRegisteredChannels();
-        fDevice->ChangeState(fair::mq::Transition::End);
+        fDevice->ChangeStateOrThrow(fair::mq::Transition::End);
         return 0;
     }
 
@@ -160,7 +160,7 @@ auto DeviceRunner::Run() -> int
     if (fConfig.Count("version")) {
         LOGV(info, verylow) << "FairMQ version: " << FAIRMQ_GIT_VERSION;
         LOGV(info, verylow) << "User device version: " << fDevice->GetVersion();
-        fDevice->ChangeState(fair::mq::Transition::End);
+        fDevice->ChangeStateOrThrow(fair::mq::Transition::End);
         return 0;
     }
 

fairmq/Error.cxx

@@ -1,39 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2019-2021 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "Error.h"
-
-namespace fair::mq {
-
-const char* ErrorCategory::name() const noexcept { return "fairmq"; }
-
-std::string ErrorCategory::message(int ev) const
-{
-    switch (static_cast<ErrorCode>(ev)) {
-        case ErrorCode::OperationInProgress:
-            return "async operation already in progress";
-        case ErrorCode::OperationTimeout:
-            return "async operation timed out";
-        case ErrorCode::OperationCanceled:
-            return "async operation canceled";
-        case ErrorCode::DeviceChangeStateFailed:
-            return "failed to change state of a fairmq device";
-        case ErrorCode::DeviceGetPropertiesFailed:
-            return "failed to get fairmq device properties";
-        case ErrorCode::DeviceSetPropertiesFailed:
-            return "failed to set fairmq device properties";
-        default:
-            return "(unrecognized error)";
-    }
-}
-
-const ErrorCategory errorCategory{};
-
-std::error_code MakeErrorCode(ErrorCode e) { return {static_cast<int>(e), errorCategory}; }
-
-}   // namespace fair::mq

fairmq/Error.h

@@ -38,14 +38,34 @@ enum class ErrorCode
     DeviceSetPropertiesFailed
 };
 
-std::error_code MakeErrorCode(ErrorCode);
-
 struct ErrorCategory : std::error_category
 {
-    const char* name() const noexcept override;
-    std::string message(int ev) const override;
+    const char* name() const noexcept override { return "fairmq"; }
+    std::string message(int ev) const override
+    {
+        switch (static_cast<ErrorCode>(ev)) {
+            case ErrorCode::OperationInProgress:
+                return "async operation already in progress";
+            case ErrorCode::OperationTimeout:
+                return "async operation timed out";
+            case ErrorCode::OperationCanceled:
+                return "async operation canceled";
+            case ErrorCode::DeviceChangeStateFailed:
+                return "failed to change state of a fairmq device";
+            case ErrorCode::DeviceGetPropertiesFailed:
+                return "failed to get fairmq device properties";
+            case ErrorCode::DeviceSetPropertiesFailed:
+                return "failed to set fairmq device properties";
+            default:
+                return "(unrecognized error)";
+        }
+    }
 };
 
+static ErrorCategory ec;
+
+inline std::error_code MakeErrorCode(ErrorCode e) { return {static_cast<int>(e), ec}; }
+
 }   // namespace fair::mq
 
 namespace std {

fairmq/Message.h

@@ -46,7 +46,7 @@ struct Message
     virtual void* GetData() const = 0;
     virtual size_t GetSize() const = 0;
 
-    virtual bool SetUsedSize(size_t size) = 0;
+    virtual bool SetUsedSize(size_t size, Alignment alignment = Alignment{0}) = 0;
 
     virtual Transport GetType() const = 0;
     TransportFactory* GetTransport() { return fTransport; }
@@ -76,6 +76,11 @@ struct MessageBadAlloc : std::runtime_error
     using std::runtime_error::runtime_error;
 };
 
+struct RefCountBadAlloc : std::runtime_error
+{
+    using std::runtime_error::runtime_error;
+};
+
 }   // namespace fair::mq
 
 using fairmq_free_fn [[deprecated("Use fair::mq::FreeFn")]] = fair::mq::FreeFn;

fairmq/Parts.h

@@ -1,5 +1,5 @@
 /********************************************************************************
- * Copyright (C) 2014-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
+ * Copyright (C) 2014-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -38,9 +38,12 @@ struct Parts
     template<typename... Ps>
     Parts(Ps&&... parts)
     {
+        fParts.reserve(sizeof...(Ps));
         AddPart(std::forward<Ps>(parts)...);
     }
 
+    [[deprecated("Avoid owning raw pointer args, use AddPart(MessagePtr) instead.")]]
+    void AddPart(Message* msg) { fParts.push_back(MessagePtr(msg)); }
     void AddPart(MessagePtr msg) { fParts.push_back(std::move(msg)); }
 
     template<typename... Ts>
@@ -60,9 +63,15 @@ struct Parts
         }
     }
 
-    reference operator[](size_type index) { return fParts[index]; }
-    const_reference operator[](size_type index) const { return fParts[index]; }
+    Message& operator[](size_type index) { return *(fParts[index]); }
+    Message const& operator[](size_type index) const { return *(fParts[index]); }
+    // TODO: For consistency with the STL interfaces, operator[] should not dereference,
+    //       but I have no good idea how to fix this.
+    // reference operator[](size_type index) { return fParts[index]; }
+    // const_reference operator[](size_type index) const { return fParts[index]; }
 
+    [[deprecated("Redundant, dereference at call site e.g. '*(parts.At(index))' instead.")]]
+    Message& AtRef(size_type index) { return *(fParts.at(index)); }
     reference At(size_type index) { return fParts.at(index); }
     const_reference At(size_type index) const { return fParts.at(index); }
 

fairmq/PluginManager.cxx

@@ -22,7 +22,6 @@ using namespace std;
 using fair::mq::Plugin;
 using fair::mq::tools::ToString;
 using fair::mq::tools::ToStrVector;
-namespace fs = boost::filesystem;
 namespace po = boost::program_options;
 namespace dll = boost::dll;
 using boost::optional;
@@ -171,7 +170,7 @@ auto fair::mq::PluginManager::LoadPluginPrelinkedDynamic(const string& pluginNam
 }
 
 auto fair::mq::PluginManager::SearchPluginFile(const string& pluginName) const
-    -> boost::filesystem::path
+    -> fs::path
 {
     for (const auto& searchPath : SearchPaths()) {
         for (const auto& libPrefix : {fgkLibPrefix, fgkLibPrefixAlt}) {
@@ -181,7 +180,7 @@ auto fair::mq::PluginManager::SearchPluginFile(const string& pluginName) const
                            libPrefix,
                            pluginName,
                            boost::dll::detail::shared_library_impl::suffix().native());
-            auto const found = boost::filesystem::exists(file);
+            auto const found = fs::exists(file);
             if (found) {
                 return file;
             }

fairmq/PluginManager.h

@@ -1,5 +1,5 @@
 /********************************************************************************
- *    Copyright (C) 2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
+ * Copyright (C) 2017-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -13,9 +13,16 @@
 #include <fairmq/PluginServices.h>
 #include <fairmq/tools/Strings.h>
 
+#if FAIRMQ_HAS_STD_FILESYSTEM
+#include <filesystem>
+namespace fs = std::filesystem;
+#else
 #define BOOST_FILESYSTEM_VERSION 3
 #define BOOST_FILESYSTEM_NO_DEPRECATED
 #include <boost/filesystem.hpp>
+namespace fs = ::boost::filesystem;
+#endif
+
 #include <boost/optional.hpp>
 #include <boost/program_options.hpp>
 #include <boost/dll/import.hpp>
@@ -60,13 +67,13 @@ class PluginManager
         LOG(debug) << "Shutting down Plugin Manager";
     }
 
-    auto SetSearchPaths(const std::vector<boost::filesystem::path>&) -> void;
-    auto AppendSearchPath(const boost::filesystem::path&) -> void;
-    auto PrependSearchPath(const boost::filesystem::path&) -> void;
-    auto SearchPaths() const -> const std::vector<boost::filesystem::path>& { return fSearchPaths; }
+    auto SetSearchPaths(const std::vector<fs::path>&) -> void;
+    auto AppendSearchPath(const fs::path&) -> void;
+    auto PrependSearchPath(const fs::path&) -> void;
+    auto SearchPaths() const -> const std::vector<fs::path>& { return fSearchPaths; }
     struct BadSearchPath : std::invalid_argument { using std::invalid_argument::invalid_argument; };
 
-    auto SearchPluginFile(const std::string&) const -> boost::filesystem::path;
+    auto SearchPluginFile(const std::string&) const -> fs::path;
     struct PluginNotFound : std::runtime_error { using std::runtime_error::runtime_error; };
     auto LoadPlugin(const std::string& pluginName) -> void;
     auto LoadPlugins(const std::vector<std::string>& pluginNames) -> void { for(const auto& pluginName : pluginNames) { LoadPlugin(pluginName); } }
@@ -88,18 +95,33 @@ class PluginManager
     auto WaitForPluginsToReleaseDeviceControl() -> void { fPluginServices->WaitForReleaseDeviceControl(); }
 
   private:
-    static auto ValidateSearchPath(const boost::filesystem::path&) -> void;
+    static auto ValidateSearchPath(const fs::path&) -> void;
 
     auto LoadPluginPrelinkedDynamic(const std::string& pluginName) -> void;
     auto LoadPluginDynamic(const std::string& pluginName) -> void;
     auto LoadPluginStatic(const std::string& pluginName) -> void;
-    template<typename... Args>
-    auto LoadSymbols(const std::string& pluginName, Args&&... args) -> void
+#if FAIRMQ_HAS_STD_FILESYSTEM
+    template<typename T>
+    static auto AdaptPathType(T&& path)
+    {
+        if constexpr(std::is_same_v<T, std::filesystem::path>) {
+            return boost::filesystem::path(std::forward<T>(path));
+        } else {
+            return std::forward<T>(path);
+        }
+    }
+#endif
+    template<typename FirstArg, typename... Args>
+    auto LoadSymbols(const std::string& pluginName, FirstArg&& farg, Args&&... args) -> void
     {
         using namespace boost::dll;
         using fair::mq::tools::ToString;
 
-        auto lib = shared_library{std::forward<Args>(args)...};
+#if FAIRMQ_HAS_STD_FILESYSTEM
+        auto lib = shared_library{AdaptPathType(std::forward<FirstArg>(farg)), std::forward<Args>(args)...};
+#else
+        auto lib = shared_library{std::forward<FirstArg>(farg), std::forward<Args>(args)...};
+#endif
         fgDLLKeepAlive.push_back(lib);
 
         fPluginFactories[pluginName] = import_alias<PluginFactory>(
@@ -121,7 +143,7 @@ class PluginManager
 
     static const std::string fgkLibPrefix;
     static const std::string fgkLibPrefixAlt;
-    std::vector<boost::filesystem::path> fSearchPaths;
+    std::vector<fs::path> fSearchPaths;
     static std::vector<boost::dll::shared_library> fgDLLKeepAlive; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
     std::map<std::string, std::function<PluginFactory>> fPluginFactories;
     std::unique_ptr<PluginServices> fPluginServices;

fairmq/Properties.cxx

@@ -1,5 +1,5 @@
 /********************************************************************************
- * Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
+ * Copyright (C) 2014-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -8,7 +8,15 @@
 
 #include <fairmq/Properties.h>
 
+#if FAIRMQ_HAS_STD_FILESYSTEM
+#include <filesystem>
+namespace fs = std::filesystem;
+#else
+#define BOOST_FILESYSTEM_VERSION 3
+#define BOOST_FILESYSTEM_NO_DEPRECATED
 #include <boost/filesystem.hpp>
+namespace fs = ::boost::filesystem;
+#endif
 
 using namespace std;
 using boost::any_cast;
@@ -84,7 +92,12 @@ unordered_map<type_index, function<pair<string, string>(const Property&)>> Prope
     { type_index(typeid(long double)),                     [](const Property& p) { return getStringPair<long double>(p, "long double"); } },
     { type_index(typeid(bool)),                            [](const Property& p) { stringstream ss; ss << boolalpha << any_cast<bool>(p); return pair<string, string>{ ss.str(), "bool" }; } },
     { type_index(typeid(vector<bool>)),                    [](const Property& p) { stringstream ss; ss << boolalpha << any_cast<vector<bool>>(p); return pair<string, string>{ ss.str(), "vector<bool>>" }; } },
-    { type_index(typeid(boost::filesystem::path)),         [](const Property& p) { return getStringPair<boost::filesystem::path>(p, "boost::filesystem::path"); } },
+    { type_index(typeid(fs::path)),                        [](const Property& p) { return getStringPair<fs::path>(p,
+#if FAIRMQ_HAS_STD_FILESYSTEM
+                                                                                                                  "std::filesystem::path"); } },
+#else
+                                                                                                                  "boost::filesystem::path"); } },
+#endif
     { type_index(typeid(vector<char>)),                    [](const Property& p) { return getStringPair<vector<char>>(p, "vector<char>"); } },
     { type_index(typeid(vector<signed char>)),             [](const Property& p) { return getStringPair<vector<signed char>>(p, "vector<signed char>"); } },
     { type_index(typeid(vector<unsigned char>)),           [](const Property& p) { return getStringPair<vector<unsigned char>>(p, "vector<unsigned char>"); } },
@@ -100,7 +113,12 @@ unordered_map<type_index, function<pair<string, string>(const Property&)>> Prope
     { type_index(typeid(vector<float>)),                   [](const Property& p) { return getStringPair<vector<float>>(p, "vector<float>"); } },
     { type_index(typeid(vector<double>)),                  [](const Property& p) { return getStringPair<vector<double>>(p, "vector<double>"); } },
     { type_index(typeid(vector<long double>)),             [](const Property& p) { return getStringPair<vector<long double>>(p, "vector<long double>"); } },
-    { type_index(typeid(vector<boost::filesystem::path>)), [](const Property& p) { return getStringPair<vector<boost::filesystem::path>>(p, "vector<boost::filesystem::path>"); } },
+    { type_index(typeid(vector<fs::path>)),                [](const Property& p) { return getStringPair<vector<fs::path>>(p,
+#if FAIRMQ_HAS_STD_FILESYSTEM
+                                                                                                                          "vector<std::filesystem::path>"); } },
+#else
+                                                                                                                          "vector<boost::filesystem::path>"); } },
+#endif
 };
 
 unordered_map<type_index, void(*)(const EventManager&, const string&, const Property&)> PropertyHelper::fEventEmitters = {
@@ -122,7 +140,7 @@ unordered_map<type_index, void(*)(const EventManager&, const string&, const Prop
     { type_index(typeid(long double)),                     [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, long double>(k, any_cast<long double>(p)); } },
     { type_index(typeid(bool)),                            [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, bool>(k, any_cast<bool>(p)); } },
     { type_index(typeid(vector<bool>)),                    [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<bool>>(k, any_cast<vector<bool>>(p)); } },
-    { type_index(typeid(boost::filesystem::path)),         [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, boost::filesystem::path>(k, any_cast<boost::filesystem::path>(p)); } },
+    { type_index(typeid(fs::path)),                        [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, fs::path>(k, any_cast<fs::path>(p)); } },
     { type_index(typeid(vector<char>)),                    [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<char>>(k, any_cast<vector<char>>(p)); } },
     { type_index(typeid(vector<signed char>)),             [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<signed char>>(k, any_cast<vector<signed char>>(p)); } },
     { type_index(typeid(vector<unsigned char>)),           [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<unsigned char>>(k, any_cast<vector<unsigned char>>(p)); } },
@@ -138,7 +156,7 @@ unordered_map<type_index, void(*)(const EventManager&, const string&, const Prop
     { type_index(typeid(vector<float>)),                   [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<float>>(k, any_cast<vector<float>>(p)); } },
     { type_index(typeid(vector<double>)),                  [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<double>>(k, any_cast<vector<double>>(p)); } },
     { type_index(typeid(vector<long double>)),             [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<long double>>(k, any_cast<vector<long double>>(p)); } },
-    { type_index(typeid(vector<boost::filesystem::path>)), [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<boost::filesystem::path>>(k, any_cast<vector<boost::filesystem::path>>(p)); } },
+    { type_index(typeid(vector<fs::path>)),                [](const EventManager& em, const string& k, const Property& p) { em.Emit<PropertyChange, vector<fs::path>>(k, any_cast<vector<fs::path>>(p)); } },
 };
 
 } // namespace fair::mq

fairmq/StateMachine.cxx

@@ -157,6 +157,13 @@ struct Machine_ : public state_machine_def<Machine_>
         }
     }
 
+    void CallStatePrep(const State state) const
+    {
+        if (!fStatePrepSignal.empty()) {
+            fStatePrepSignal(state);
+        }
+    }
+
     void CallNewTransitionCallbacks(const Transition transition) const
     {
         if (!fNewTransitionSignal.empty()) {
@@ -170,11 +177,13 @@ struct Machine_ : public state_machine_def<Machine_>
     atomic<bool> fLastTransitionResult;
 
     mutex fStateMtx;
+    mutex fSubscriptionsMtx;
     atomic<bool> fNewStatePending;
     condition_variable fNewStatePendingCV;
 
     boost::signals2::signal<void(const State)> fStateChangeSignal;
     boost::signals2::signal<void(const State)> fStateHandleSignal;
+    boost::signals2::signal<void(const State)> fStatePrepSignal;
     boost::signals2::signal<void(const Transition)> fNewTransitionSignal;
     unordered_map<string, boost::signals2::connection> fStateChangeSignalsMap;
     unordered_map<string, boost::signals2::connection> fNewTransitionSignalsMap;
@@ -198,6 +207,7 @@ struct Machine_ : public state_machine_def<Machine_>
                 }
             }
 
+            CallStatePrep(fState);
             CallStateChangeCallbacks(fState);
             CallStateHandler(fState);
         }
@@ -301,18 +311,33 @@ try {
 
 void StateMachine::SubscribeToStateChange(const string& key, function<void(const State)> callback)
 {
-    static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignalsMap.insert({key, static_pointer_cast<FairMQFSM>(fFsm)->fStateChangeSignal.connect(callback)});
+    // Check if the key has a integer value as prefix, if yes, decode it.
+    int i = strtol(key.c_str(), nullptr, 10);
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    lock_guard<mutex> lock(fsm->fSubscriptionsMtx);
+    fsm->fStateChangeSignalsMap.insert({key, fsm->fStateChangeSignal.connect(i, callback)});
 }
 
 void StateMachine::UnsubscribeFromStateChange(const string& key)
 {
     auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    lock_guard<mutex> lock(fsm->fSubscriptionsMtx);
     if (fsm->fStateChangeSignalsMap.count(key)) {
         fsm->fStateChangeSignalsMap.at(key).disconnect();
         fsm->fStateChangeSignalsMap.erase(key);
     }
 }
 
+void StateMachine::PrepareState(std::function<void(const State)> callback)
+{
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    if (fsm->fStatePrepSignal.empty()) {
+        fsm->fStatePrepSignal.connect(callback);
+    } else {
+        LOG(error) << "state preparation handler is already set";
+    }
+}
+
 void StateMachine::HandleStates(function<void(const State)> callback)
 {
     auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
@@ -326,6 +351,9 @@ void StateMachine::HandleStates(function<void(const State)> callback)
 void StateMachine::StopHandlingStates()
 {
     auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    if (!fsm->fStatePrepSignal.empty()) {
+        fsm->fStatePrepSignal.disconnect_all_slots();
+    }
     if (!fsm->fStateHandleSignal.empty()) {
         fsm->fStateHandleSignal.disconnect_all_slots();
     }
@@ -333,12 +361,15 @@ void StateMachine::StopHandlingStates()
 
 void StateMachine::SubscribeToNewTransition(const string& key, function<void(const Transition)> callback)
 {
-    static_pointer_cast<FairMQFSM>(fFsm)->fNewTransitionSignalsMap.insert({key, static_pointer_cast<FairMQFSM>(fFsm)->fNewTransitionSignal.connect(callback)});
+    auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    lock_guard<mutex> lock(fsm->fSubscriptionsMtx);
+    fsm->fNewTransitionSignalsMap.insert({key, fsm->fNewTransitionSignal.connect(callback)});
 }
 
 void StateMachine::UnsubscribeFromNewTransition(const string& key)
 {
     auto fsm = static_pointer_cast<FairMQFSM>(fFsm);
+    lock_guard<mutex> lock(fsm->fSubscriptionsMtx);
     if (fsm->fNewTransitionSignalsMap.count(key)) {
         fsm->fNewTransitionSignalsMap.at(key).disconnect();
         fsm->fNewTransitionSignalsMap.erase(key);

fairmq/StateMachine.h

@@ -35,6 +35,7 @@ class StateMachine
     void SubscribeToStateChange(const std::string& key, std::function<void(const State)> callback);
     void UnsubscribeFromStateChange(const std::string& key);
 
+    void PrepareState(std::function<void(const State)> callback);
     void HandleStates(std::function<void(const State)> callback);
     void StopHandlingStates();
 

fairmq/Tools.h

@@ -1,5 +1,5 @@
 /********************************************************************************
- * Copyright (C) 2017-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
+ * Copyright (C) 2017-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -10,6 +10,7 @@
 #define FAIR_MQ_TOOLS_H
 
 // IWYU pragma: begin_exports
+#include <fairmq/tools/Compiler.h>
 #include <fairmq/tools/CppSTL.h>
 #include <fairmq/tools/Exceptions.h>
 #include <fairmq/tools/InstanceLimit.h>

fairmq/TransportFactory.cxx

@@ -1,5 +1,5 @@
 /********************************************************************************
- * Copyright (C) 2017-2021 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
+ * Copyright (C) 2017-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -9,9 +9,6 @@
 #include <fairmq/TransportFactory.h>
 #include <fairmq/shmem/TransportFactory.h>
 #include <fairmq/zeromq/TransportFactory.h>
-#ifdef BUILD_OFI_TRANSPORT
-#include <fairmq/ofi/TransportFactory.h>
-#endif
 #include <fairlogger/Logger.h>
 #include <fairmq/Tools.h>
 #include <memory>
@@ -39,20 +36,12 @@ auto TransportFactory::CreateTransportFactory(const string& type,
     } else if (type == "shmem") {
         return make_shared<shmem::TransportFactory>(finalId, config);
     }
-#ifdef BUILD_OFI_TRANSPORT
-    else if (type == "ofi") {
-        return make_shared<ofi::TransportFactory>(finalId, config);
-    }
-#endif /* BUILD_OFI_TRANSPORT */
     else {
         LOG(error) << "Unavailable transport requested: "
                    << "\"" << type << "\""
                    << ". Available are: "
                    << "\"zeromq\","
                    << "\"shmem\""
-#ifdef BUILD_OFI_TRANSPORT
-                   << ", and \"ofi\""
-#endif /* BUILD_OFI_TRANSPORT */
                    << ". Exiting.";
         throw TransportFactoryError(tools::ToString("Unavailable transport requested: ", type));
     }

fairmq/Transports.h

@@ -1,5 +1,5 @@
 /********************************************************************************
- * Copyright (C) 2014-2021 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
+ * Copyright (C) 2014-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -22,8 +22,7 @@ enum class Transport
 {
     DEFAULT,
     ZMQ,
-    SHM,
-    OFI
+    SHM
 };
 
 struct TransportError : std::runtime_error
@@ -34,15 +33,13 @@ struct TransportError : std::runtime_error
 static const std::unordered_map<std::string, Transport> TransportTypes{
     {"default", Transport::DEFAULT},
     {"zeromq", Transport::ZMQ},
-    {"shmem", Transport::SHM},
-    {"ofi", Transport::OFI}
+    {"shmem", Transport::SHM}
 };
 
 static const std::unordered_map<Transport, std::string> TransportNames{
     {Transport::DEFAULT, "default"},
     {Transport::ZMQ, "zeromq"},
-    {Transport::SHM, "shmem"},
-    {Transport::OFI, "ofi"}
+    {Transport::SHM, "shmem"}
 };
 
 inline std::string TransportName(Transport transport) { return TransportNames.at(transport); }
@@ -61,11 +58,7 @@ inline std::ostream& operator<<(std::ostream& os, const Transport& transport)
 
 inline auto GetEnabledTransports() -> std::vector<Transport>
 {
-#ifdef BUILD_OFI_TRANSPORT
-    return {Transport::ZMQ, Transport::SHM, Transport::OFI};
-#else
     return {Transport::ZMQ, Transport::SHM};
-#endif
 }
 
 }   // namespace fair::mq

fairmq/UnmanagedRegion.h

@@ -134,6 +134,7 @@ struct RegionConfig
     int creationFlags = 0; /// flags passed to the underlying transport on region creation
     int64_t userFlags = 0; /// custom flags that have no effect on the transport, but can be retrieved from the region by the user
     uint64_t size = 0; /// region size
+    uint64_t rcSegmentSize = 100000000; /// size of the segment that stores reference counts when "soft"-copying the messages
     std::string path = ""; /// file path, if the region is backed by a file
     std::optional<uint16_t> id = std::nullopt; /// region id
     uint32_t linger = 100; /// delay in ms before region destruction to collect outstanding events

fairmq/Version.h.in

@@ -19,7 +19,7 @@
 #define FAIRMQ_GIT_DATE "@PROJECT_GIT_DATE@"
 #define FAIRMQ_REPO_URL "https://github.com/FairRootGroup/FairMQ"
 #define FAIRMQ_LICENSE "LGPL-3.0"
-#define FAIRMQ_COPYRIGHT "2012-2022 GSI"
+#define FAIRMQ_COPYRIGHT "2012-2023 GSI"
 #define FAIRMQ_BUILD_TYPE "@CMAKE_BUILD_TYPE@"
 
 #endif // FAIR_MQ_VERSION_H

fairmq/devices/Merger.h

@@ -1,5 +1,5 @@
 /********************************************************************************
- * Copyright (C) 2014-2021 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
+ * Copyright (C) 2014-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -47,7 +47,7 @@ class Merger : public Device
 
         std::vector<Channel*> chans;
 
-        for (auto& chan : fChannels.at(fInChannelName)) {
+        for (auto& chan : GetChannels().at(fInChannelName)) {
             chans.push_back(&chan);
         }
 

fairmq/devices/Multiplier.h

@@ -1,5 +1,5 @@
 /********************************************************************************
- * Copyright (C) 2014-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
+ * Copyright (C) 2014-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *

fairmq/devices/startMQBenchmark.sh.in

@@ -63,6 +63,7 @@ echo "Usage: startBenchmark [message size=1000000] [number of iterations=0] [tra
 
 SAMPLER="fairmq-bsampler"
 SAMPLER+=" --id bsampler1"
+SAMPLER+=" --shm-monitor true"
 #SAMPLER+=" --io-threads 2"
 #SAMPLER+=" --control static"
 SAMPLER+=" --transport $transport"
@@ -71,6 +72,7 @@ SAMPLER+=" --msg-size $msgSize"
 SAMPLER+=" --multipart $multipart"
 SAMPLER+=" --num-parts $numParts"
 SAMPLER+=" --shm-throw-bad-alloc false"
+# SAMPLER+=" --shm-metadata-msg-size 1024"
 # SAMPLER+=" --msg-rate 1000"
 SAMPLER+=" --max-iterations $maxIterations"
 SAMPLER+=" --channel-config name=data,type=pair,method=bind,address=tcp://127.0.0.1:5555"
@@ -81,6 +83,7 @@ echo "pid: $!"
 
 SINK="fairmq-sink"
 SINK+=" --id sink1"
+SINK+=" --shm-monitor true"
 #SINK+=" --io-threads 2"
 #SINK+=" --control static"
 SINK+=" --transport $transport"

fairmq/ofi/Context.cxx

@@ -1,130 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2018-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include <arpa/inet.h>
-#include <asiofi/version.hpp>
-#include <cassert>
-#include <cstring>
-#include <fairlogger/Logger.h>
-#include <fairmq/ofi/Context.h>
-#include <fairmq/tools/Strings.h>
-#include <memory>
-#include <netinet/in.h>
-#include <regex>
-#include <string>
-#include <sys/socket.h>
-
-namespace fair::mq::ofi
-{
-
-using namespace std;
-
-Context::Context(mq::TransportFactory& sendFactory,
-                 mq::TransportFactory& receiveFactory,
-                 int numberIoThreads)
-    : fIoWork(fIoContext)
-    , fReceiveFactory(receiveFactory)
-    , fSendFactory(sendFactory)
-    , fSizeHint(0)
-{
-    InitThreadPool(numberIoThreads);
-}
-
-auto Context::InitThreadPool(int numberIoThreads) -> void
-{
-    assert(numberIoThreads > 0);
-
-    for (int i = 1; i <= numberIoThreads; ++i) {
-        fThreadPool.emplace_back([&, i, numberIoThreads]{
-            try {
-                LOG(debug) << "OFI transport: I/O thread #" << i << " of " << numberIoThreads << " started";
-                fIoContext.run();
-                LOG(debug) << "OFI transport: I/O thread #" << i << " of " << numberIoThreads << " stopped";
-            } catch (const std::exception& e) {
-                LOG(error) << "OFI transport: Uncaught exception in I/O thread #" << i << ": " << e.what();
-            } catch (...) {
-                LOG(error) << "OFI transport: Uncaught exception in I/O thread #" << i;
-            }
-        });
-    }
-}
-
-auto Context::Reset() -> void
-{
-    // TODO "Linger", rethink this
-    std::this_thread::sleep_for(std::chrono::milliseconds(1000));
-
-    fIoContext.stop();
-}
-
-Context::~Context()
-{
-    for (auto& thread : fThreadPool)
-        thread.join();
-}
-
-auto Context::GetAsiofiVersion() const -> string
-{
-    return ASIOFI_VERSION;
-}
-
-auto Context::ConvertAddress(std::string address) -> Address
-{
-    string protocol, ip;
-    unsigned int port = 0;
-    regex address_regex("^([a-z]+)://([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+):([0-9]+).*");
-    smatch address_result;
-    if (regex_match(address, address_result, address_regex)) {
-        protocol = address_result[1];
-        ip = address_result[2];
-        port = stoul(address_result[3]);
-        // LOG(debug) << "Parsed '" << protocol << "', '" << ip << "', '" << port << "' fields from '" << address << "'";
-    } else {
-        throw ContextError(tools::ToString("Wrong format: Address must be in format prot://ip:port"));
-    }
-
-    return { protocol, ip, port };
-}
-
-auto Context::ConvertAddress(Address address) -> sockaddr_in
-{
-    sockaddr_in sa;
-    if (inet_pton(AF_INET, address.Ip.c_str(), &(sa.sin_addr)) != 1)
-        throw ContextError(tools::ToString("Failed to convert given IP '", address.Ip, "' to struct in_addr, reason: ", strerror(errno)));
-    sa.sin_port = htons(address.Port);
-    sa.sin_family = AF_INET;
-
-    return sa;
-}
-
-auto Context::ConvertAddress(sockaddr_in address) -> Address
-{
-    return {"tcp", inet_ntoa(address.sin_addr), ntohs(address.sin_port)};
-}
-
-auto Context::VerifyAddress(const std::string& address) -> Address
-{
-    auto addr = ConvertAddress(address);
-
-    if (!(addr.Protocol == "tcp" || addr.Protocol == "verbs"))
-        throw ContextError("Wrong protocol: Supported protocols are: tcp:// and verbs://");
-
-    return addr;
-}
-
-auto Context::MakeReceiveMessage(size_t size) -> MessagePtr
-{
-    return fReceiveFactory.CreateMessage(size);
-}
-
-auto Context::MakeSendMessage(size_t size) -> MessagePtr
-{
-    return fSendFactory.CreateMessage(size);
-}
-
-} // namespace fair::mq::ofi

fairmq/ofi/Context.h

@@ -1,92 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2018-2021 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef FAIR_MQ_OFI_CONTEXT_H
-#define FAIR_MQ_OFI_CONTEXT_H
-
-#include <asio/io_context.hpp>
-#include <asiofi/domain.hpp>
-#include <asiofi/fabric.hpp>
-#include <asiofi/info.hpp>
-#include <fairlogger/Logger.h>
-#include <fairmq/TransportFactory.h>
-#include <memory>
-#include <netinet/in.h>
-#include <ostream>
-#include <stdexcept>
-#include <string>
-#include <thread>
-#include <vector>
-
-namespace fair::mq::ofi
-{
-
-enum class ConnectionType : bool { Bind, Connect };
-
-struct Address {
-    std::string Protocol;
-    std::string Ip;
-    unsigned int Port;
-    friend auto operator<<(std::ostream& os, const Address& a) -> std::ostream&
-    {
-        return os << a.Protocol << "://" << a.Ip << ":" << a.Port;
-    }
-    friend auto operator==(const Address& lhs, const Address& rhs) -> bool
-    {
-        return (lhs.Protocol == rhs.Protocol) && (lhs.Ip == rhs.Ip) && (lhs.Port == rhs.Port);
-    }
-};
-
-/**
- * @class Context Context.h <fairmq/ofi/Context.h>
- * @brief Transport-wide context
- *
- * @todo TODO insert long description
- */
-class Context
-{
-  public:
-    Context(mq::TransportFactory& sendFactory,
-            mq::TransportFactory& receiveFactory,
-            int numberIoThreads = 1);
-    Context(const Context&) = delete;
-    Context(Context&&) = delete;
-    Context& operator=(const Context&) = delete;
-    Context& operator=(Context&&) = delete;
-    ~Context();
-
-    auto GetAsiofiVersion() const -> std::string;
-    auto GetIoContext() -> asio::io_context& { return fIoContext; }
-    static auto ConvertAddress(std::string address) -> Address;
-    static auto ConvertAddress(Address address) -> sockaddr_in;
-    static auto ConvertAddress(sockaddr_in address) -> Address;
-    static auto VerifyAddress(const std::string& address) -> Address;
-    auto Interrupt() -> void { LOG(debug) << "OFI transport: Interrupted (NOOP - not implemented)."; }
-    auto Resume() -> void { LOG(debug) << "OFI transport: Resumed (NOOP - not implemented)."; }
-    auto Reset() -> void;
-    auto MakeReceiveMessage(size_t size) -> MessagePtr;
-    auto MakeSendMessage(size_t size) -> MessagePtr;
-    auto GetSizeHint() -> size_t { return fSizeHint; }
-    auto SetSizeHint(size_t size) -> void { fSizeHint = size; }
-
-  private:
-    asio::io_context fIoContext;
-    asio::io_context::work fIoWork;
-    std::vector<std::thread> fThreadPool;
-    mq::TransportFactory& fReceiveFactory;
-    mq::TransportFactory& fSendFactory;
-    size_t fSizeHint;
-
-    auto InitThreadPool(int numberIoThreads) -> void;
-}; /* class Context */
-
-struct ContextError : std::runtime_error { using std::runtime_error::runtime_error; };
-
-} // namespace fair::mq::ofi
-
-#endif /* FAIR_MQ_OFI_CONTEXT_H */

fairmq/ofi/ControlMessages.h

@@ -1,112 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2018-2021 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef FAIR_MQ_OFI_CONTROLMESSAGES_H
-#define FAIR_MQ_OFI_CONTROLMESSAGES_H
-
-#include <asio/buffer.hpp>
-#include <cstdint>
-#include <fairlogger/Logger.h>
-#include <functional>
-#include <memory>
-#include <memory_resource>
-#include <type_traits>
-
-namespace asio
-{
-
-template<typename PodType>
-auto buffer(const PodType& obj) -> asio::const_buffer
-{
-    return asio::const_buffer(static_cast<const void*>(&obj), sizeof(PodType));
-}
-
-} // namespace asio
-
-namespace fair::mq::ofi
-{
-
-enum class ControlMessageType
-{
-    Empty = 1,
-    PostBuffer,
-    PostMultiPartStartBuffer
-};
-
-struct Empty
-{};
-
-struct PostBuffer
-{
-    uint64_t size;   // buffer size (size_t)
-};
-
-struct PostMultiPartStartBuffer
-{
-    uint32_t numParts;   // buffer size (size_t)
-    uint64_t size;       // buffer size (size_t)
-};
-
-union ControlMessageContent
-{
-    PostBuffer postBuffer;
-    PostMultiPartStartBuffer postMultiPartStartBuffer;
-};
-
-struct ControlMessage
-{
-    ControlMessageType type;
-    ControlMessageContent msg;
-};
-
-template<typename T>
-using unique_ptr = std::unique_ptr<T, std::function<void(T*)>>;
-
-template<typename T, typename... Args>
-auto MakeControlMessageWithPmr(std::pmr::memory_resource& pmr, Args&&... args)
-    -> ofi::unique_ptr<ControlMessage>
-{
-    void* mem = pmr.allocate(sizeof(ControlMessage));
-    ControlMessage* ctrl = new (mem) ControlMessage();
-
-    if (std::is_same<T, PostBuffer>::value) {
-        ctrl->type = ControlMessageType::PostBuffer;
-        ctrl->msg.postBuffer = PostBuffer(std::forward<Args>(args)...);
-    } else if (std::is_same<T, PostMultiPartStartBuffer>::value) {
-        ctrl->type = ControlMessageType::PostMultiPartStartBuffer;
-        ctrl->msg.postMultiPartStartBuffer = PostMultiPartStartBuffer(std::forward<Args>(args)...);
-    } else if (std::is_same<T, Empty>::value) {
-        ctrl->type = ControlMessageType::Empty;
-    }
-
-    return ofi::unique_ptr<ControlMessage>(ctrl, [&pmr](ControlMessage* p) {
-        p->~ControlMessage();
-        pmr.deallocate(p, sizeof(T));
-    });
-}
-
-template<typename T, typename... Args>
-auto MakeControlMessage(Args&&... args) -> ControlMessage
-{
-    ControlMessage ctrl;
-
-    if (std::is_same<T, PostBuffer>::value) {
-        ctrl.type = ControlMessageType::PostBuffer;
-    } else if (std::is_same<T, PostMultiPartStartBuffer>::value) {
-        ctrl.type = ControlMessageType::PostMultiPartStartBuffer;
-    } else if (std::is_same<T, Empty>::value) {
-        ctrl.type = ControlMessageType::Empty;
-    }
-    ctrl.msg = T(std::forward<Args>(args)...);
-
-    return ctrl;
-}
-
-} // namespace fair::mq::ofi
-
-#endif /* FAIR_MQ_OFI_CONTROLMESSAGES_H */

fairmq/ofi/Message.cxx

@@ -1,199 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2018-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include <asiofi.hpp>
-#include <cassert>
-#include <cstdlib>
-#include <fairlogger/Logger.h>
-#include <fairmq/ofi/Message.h>
-#include <zmq.h>
-
-namespace fair::mq::ofi
-{
-
-using namespace std;
-
-Message::Message(pmr::memory_resource* pmr)
-    : fInitialSize(0)
-    , fSize(0)
-    , fData(nullptr)
-    , fFreeFunction(nullptr)
-    , fHint(nullptr)
-    , fPmr(pmr)
-{
-}
-
-Message::Message(pmr::memory_resource* pmr, Alignment /* alignment */)
-    : fInitialSize(0)
-    , fSize(0)
-    , fData(nullptr)
-    , fFreeFunction(nullptr)
-    , fHint(nullptr)
-    , fPmr(pmr)
-{
-}
-
-Message::Message(pmr::memory_resource* pmr, const size_t size)
-    : fInitialSize(size)
-    , fSize(size)
-    , fData(nullptr)
-    , fFreeFunction(nullptr)
-    , fHint(nullptr)
-    , fPmr(pmr)
-{
-    if (size) {
-        fData = fPmr->allocate(size);
-        assert(fData);
-    }
-}
-
-Message::Message(pmr::memory_resource* pmr, const size_t size, Alignment /* alignment */)
-    : fInitialSize(size)
-    , fSize(size)
-    , fData(nullptr)
-    , fFreeFunction(nullptr)
-    , fHint(nullptr)
-    , fPmr(pmr)
-{
-    if (size) {
-        fData = fPmr->allocate(size);
-        assert(fData);
-    }
-}
-
-Message::Message(pmr::memory_resource* pmr,
-                 void* data,
-                 const size_t size,
-                 FreeFn* ffn,
-                 void* hint)
-    : fInitialSize(size)
-    , fSize(size)
-    , fData(data)
-    , fFreeFunction(ffn)
-    , fHint(hint)
-    , fPmr(pmr)
-{}
-
-Message::Message(pmr::memory_resource* /*pmr*/,
-                 fair::mq::UnmanagedRegionPtr& /*region*/,
-                 void* /*data*/,
-                 const size_t /*size*/,
-                 void* /*hint*/)
-{
-    throw MessageError{"Not yet implemented."};
-}
-
-auto Message::Rebuild() -> void
-{
-    if (fFreeFunction) {
-        fFreeFunction(fData, fHint);
-    } else {
-        if (fData) {
-            fPmr->deallocate(fData, fSize);
-        }
-    }
-    fData = nullptr;
-    fInitialSize = 0;
-    fSize = 0;
-    fFreeFunction = nullptr;
-    fHint = nullptr;
-}
-
-auto Message::Rebuild(Alignment /* alignment */) -> void
-{
-    // TODO: implement alignment
-    Rebuild();
-}
-
-auto Message::Rebuild(size_t size) -> void
-{
-    if (fFreeFunction) {
-      fFreeFunction(fData, fHint);
-    } else {
-        if (fData) {
-            fPmr->deallocate(fData, fSize);
-        }
-    }
-    if (size) {
-        fData = fPmr->allocate(size);
-        assert(fData);
-    } else {
-        fData = nullptr;
-    }
-    fInitialSize = size;
-    fSize = size;
-    fFreeFunction = nullptr;
-    fHint = nullptr;
-}
-
-auto Message::Rebuild(size_t size, Alignment /* alignment */) -> void
-{
-    // TODO: implement alignment
-    Rebuild(size);
-}
-
-auto Message::Rebuild(void* /*data*/, size_t size, FreeFn* ffn, void* hint) -> void
-{
-    if (fFreeFunction) {
-      fFreeFunction(fData, fHint);
-    } else {
-        if (fData) {
-            fPmr->deallocate(fData, fSize);
-        }
-    }
-    if (size) {
-        fData = fPmr->allocate(size);
-        assert(fData);
-    } else {
-        fData = nullptr;
-    }
-    assert(fData);
-    fInitialSize = size;
-    fSize = size;
-    fFreeFunction = ffn;
-    fHint = hint;
-}
-
-auto Message::GetData() const -> void*
-{
-    return fData;
-}
-
-auto Message::GetSize() const -> size_t
-{
-    return fSize;
-}
-
-auto Message::SetUsedSize(size_t size) -> bool
-{
-    if (size == fSize) {
-        return true;
-    } else if (size <= fSize) {
-        throw MessageError{"Message size shrinking not yet implemented."};
-    } else {
-        throw MessageError{"Cannot grow message size."};
-    }
-}
-
-auto Message::Copy(const fair::mq::Message& /*msg*/) -> void
-{
-    throw MessageError{"Not yet implemented."};
-}
-
-Message::~Message()
-{
-    if (fFreeFunction) {
-        fFreeFunction(fData, fHint);
-    } else {
-        if (fData) {
-            fPmr->deallocate(fData, fSize);
-        }
-    }
-}
-
-} // namespace fair::mq::ofi

fairmq/ofi/Message.h

@@ -1,81 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2018-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef FAIR_MQ_OFI_MESSAGE_H
-#define FAIR_MQ_OFI_MESSAGE_H
-
-#include <asiofi.hpp>
-#include <atomic>
-#include <cstddef> // size_t
-#include <fairmq/Message.h>
-#include <fairmq/Transports.h>
-#include <fairmq/UnmanagedRegion.h>
-#include <memory_resource>
-#include <zmq.h>
-
-namespace fair::mq::ofi
-{
-
-/**
- * @class Message Message.h <fairmq/ofi/Message.h>
- * @brief
- *
- * @todo TODO insert long description
- */
-class Message final : public fair::mq::Message
-{
-  public:
-    Message(std::pmr::memory_resource* pmr);
-    Message(std::pmr::memory_resource* pmr, Alignment alignment);
-    Message(std::pmr::memory_resource* pmr, size_t size);
-    Message(std::pmr::memory_resource* pmr, size_t size, Alignment alignment);
-    Message(std::pmr::memory_resource* pmr,
-            void* data,
-            size_t size,
-            FreeFn* ffn,
-            void* hint = nullptr);
-    Message(std::pmr::memory_resource* pmr,
-            fair::mq::UnmanagedRegionPtr& region,
-            void* data,
-            size_t size,
-            void* hint = 0);
-
-    Message(const Message&) = delete;
-    Message(Message&&) = delete;
-    Message& operator=(const Message&) = delete;
-    Message& operator=(Message&&) = delete;
-
-    auto Rebuild() -> void override;
-    auto Rebuild(Alignment alignment) -> void override;
-    auto Rebuild(size_t size) -> void override;
-    auto Rebuild(size_t size, Alignment alignment) -> void override;
-    auto Rebuild(void* data, size_t size, FreeFn* ffn, void* hint = nullptr) -> void override;
-
-    auto GetData() const -> void* override;
-    auto GetSize() const -> size_t override;
-
-    auto SetUsedSize(size_t size) -> bool override;
-
-    auto GetType() const -> fair::mq::Transport override { return fair::mq::Transport::OFI; }
-
-    auto Copy(const fair::mq::Message& msg) -> void override;
-
-    ~Message() override;
-
-  private:
-    size_t fInitialSize;
-    size_t fSize;
-    void* fData;
-    FreeFn* fFreeFunction;
-    void* fHint;
-    std::pmr::memory_resource* fPmr;
-}; /* class Message */
-
-} // namespace fair::mq::ofi
-
-#endif /* FAIR_MQ_OFI_MESSAGE_H */

fairmq/ofi/Socket.cxx

@@ -1,680 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2018-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include <fairmq/ofi/ControlMessages.h>
-#include <fairmq/ofi/Socket.h>
-#include <fairmq/ofi/TransportFactory.h>
-#include <fairmq/tools/Strings.h>
-#include <fairlogger/Logger.h>
-
-#include <asiofi.hpp>
-#include <asio/buffer.hpp>
-#include <asio/dispatch.hpp>
-#include <asio/post.hpp>
-#include <chrono>
-#include <cstring>
-#include <functional>
-#include <memory>
-#include <sstream>
-#include <thread>
-#include <mutex>
-#include <queue>
-
-namespace fair::mq::ofi
-{
-
-using namespace std;
-
-Socket::Socket(Context& context, const string& type, const string& name, const string& id /*= ""*/)
-    : fContext(context)
-    , fOfiInfo(nullptr)
-    , fOfiFabric(nullptr)
-    , fOfiDomain(nullptr)
-    , fPassiveEndpoint(nullptr)
-    , fDataEndpoint(nullptr)
-    , fControlEndpoint(nullptr)
-    , fId(id + "." + name + "." + type)
-    , fBytesTx(0)
-    , fBytesRx(0)
-    , fMessagesTx(0)
-    , fMessagesRx(0)
-    , fMultiPartRecvCounter(-1)
-    , fSendPushSem(fContext.GetIoContext(), 384)
-    , fSendPopSem(fContext.GetIoContext(), 0)
-    , fRecvPushSem(fContext.GetIoContext(), 384)
-    , fRecvPopSem(fContext.GetIoContext(), 0)
-    , fNeedOfiMemoryRegistration(false)
-{
-    if (type != "pair") {
-        throw SocketError{tools::ToString("Socket type '", type, "' not implemented for ofi transport.")};
-    }
-}
-
-auto Socket::InitOfi(Address addr) -> void
-{
-    if (!fOfiInfo) {
-      assert(!fOfiFabric);
-      assert(!fOfiDomain);
-
-      asiofi::hints hints;
-      if (addr.Protocol == "tcp") {
-          hints.set_provider("sockets");
-      } else if (addr.Protocol == "verbs") {
-          hints.set_provider("verbs");
-      }
-      if (fRemoteAddr == addr) {
-          fOfiInfo = make_unique<asiofi::info>(addr.Ip.c_str(), std::to_string(addr.Port).c_str(), 0, hints);
-      } else {
-          fOfiInfo = make_unique<asiofi::info>(addr.Ip.c_str(), std::to_string(addr.Port).c_str(), FI_SOURCE, hints);
-      }
-
-      LOG(debug) << "OFI transport (" << fId << "): " << *fOfiInfo;
-
-      fOfiFabric = make_unique<asiofi::fabric>(*fOfiInfo);
-
-      fOfiDomain = make_unique<asiofi::domain>(*fOfiFabric);
-    }
-}
-
-auto Socket::Bind(const string& addr) -> bool
-try {
-    fLocalAddr = Context::VerifyAddress(addr);
-    if (fLocalAddr.Protocol == "verbs") {
-        fNeedOfiMemoryRegistration = true;
-    }
-
-    InitOfi(fLocalAddr);
-
-    fPassiveEndpoint = make_unique<asiofi::passive_endpoint>(fContext.GetIoContext(), *fOfiFabric);
-    //fPassiveEndpoint->set_local_address(Context::ConvertAddress(fLocalAddr));
-
-    BindControlEndpoint();
-
-    return true;
-}
-// TODO catch the correct ofi error
-catch (const SilentSocketError& e)
-{
-    // do not print error in this case, this is handled by fair::mq::Device
-    // in case no connection could be established after trying a number of random ports from a range.
-    return false;
-}
-catch (const std::exception& e)
-{
-    LOG(error) << "OFI transport: " << e.what();
-    return false;
-}
-catch (...)
-{
-    LOG(error) << "OFI transport: Unknown exception in ofi::Socket::Bind";
-    return false;
-}
-
-auto Socket::BindControlEndpoint() -> void
-{
-    assert(!fControlEndpoint);
-
-    fPassiveEndpoint->listen([&](asiofi::info&& info) {
-        LOG(debug) << "OFI transport (" << fId
-                   << "): control band connection request received. Accepting ...";
-        fControlEndpoint = make_unique<asiofi::connected_endpoint>(
-            fContext.GetIoContext(), *fOfiDomain, info);
-        fControlEndpoint->enable();
-        fControlEndpoint->accept([&]() {
-            LOG(debug) << "OFI transport (" << fId << "): control band connection accepted.";
-
-            BindDataEndpoint();
-        });
-    });
-
-    LOG(debug) << "OFI transport (" << fId << "): control band bound to " << fLocalAddr;
-}
-
-auto Socket::BindDataEndpoint() -> void
-{
-    assert(!fDataEndpoint);
-
-    fPassiveEndpoint->listen([&](asiofi::info&& info) {
-        LOG(debug) << "OFI transport (" << fId
-                   << "): data band connection request received. Accepting ...";
-        fDataEndpoint = make_unique<asiofi::connected_endpoint>(
-            fContext.GetIoContext(), *fOfiDomain, info);
-        fDataEndpoint->enable();
-        fDataEndpoint->accept([&]() {
-            LOG(debug) << "OFI transport (" << fId << "): data band connection accepted.";
-
-            if (fContext.GetSizeHint()) {
-                asio::post(fContext.GetIoContext(),
-                                  std::bind(&Socket::SendQueueReaderStatic, this));
-                asio::post(fContext.GetIoContext(),
-                                  std::bind(&Socket::RecvQueueReaderStatic, this));
-            } else {
-                asio::post(fContext.GetIoContext(),
-                                  std::bind(&Socket::SendQueueReader, this));
-                asio::post(fContext.GetIoContext(),
-                                  std::bind(&Socket::RecvControlQueueReader, this));
-            }
-        });
-    });
-
-    LOG(debug) << "OFI transport (" << fId << "): data band bound to " << fLocalAddr;
-}
-
-auto Socket::Connect(const string& address) -> bool
-try {
-    fRemoteAddr = Context::VerifyAddress(address);
-    if (fRemoteAddr.Protocol == "verbs") {
-        fNeedOfiMemoryRegistration = true;
-    }
-
-    InitOfi(fRemoteAddr);
-
-    ConnectEndpoint(fControlEndpoint, Band::Control);
-    ConnectEndpoint(fDataEndpoint, Band::Data);
-
-    if (fContext.GetSizeHint()) {
-        asio::post(fContext.GetIoContext(), std::bind(&Socket::SendQueueReaderStatic, this));
-        asio::post(fContext.GetIoContext(), std::bind(&Socket::RecvQueueReaderStatic, this));
-    } else {
-        asio::post(fContext.GetIoContext(), std::bind(&Socket::SendQueueReader, this));
-        asio::post(fContext.GetIoContext(), std::bind(&Socket::RecvControlQueueReader, this));
-    }
-
-    return true;
-}
-catch (const SilentSocketError& e)
-{
-    // do not print error in this case, this is handled by fair::mq::Device
-    return false;
-}
-catch (const std::exception& e)
-{
-    LOG(error) << "OFI transport: " << e.what();
-    return false;
-}
-catch (...)
-{
-    LOG(error) << "OFI transport: Unknown exception in ofi::Socket::Connect";
-    return false;
-}
-
-auto Socket::ConnectEndpoint(std::unique_ptr<asiofi::connected_endpoint>& endpoint, Band type) -> void
-{
-    assert(!endpoint);
-
-    std::string band(type == Band::Control ? "control" : "data");
-
-    endpoint = make_unique<asiofi::connected_endpoint>(fContext.GetIoContext(), *fOfiDomain);
-    endpoint->enable();
-
-    LOG(debug) << "OFI transport (" << fId << "): Sending " << band << " band connection request to " << fRemoteAddr;
-
-    std::mutex mtx;
-    std::condition_variable cv;
-    bool notified(false), connected(false);
-
-    while (true) {
-        endpoint->connect(Context::ConvertAddress(fRemoteAddr), [&, band](asiofi::eq::event event) {
-            // LOG(debug) << "OFI transport (" << fId << "): " << band << " band conn event happened";
-            std::unique_lock<std::mutex> lk2(mtx);
-            notified = true;
-            if (event == asiofi::eq::event::connected) {
-                LOG(debug) << "OFI transport (" << fId << "): " << band << " band connected.";
-                connected = true;
-            } else {
-                // LOG(debug) << "OFI transport (" << fId << "): " << band << " band connection refused. Trying again.";
-            }
-            lk2.unlock();
-            cv.notify_one();
-        });
-
-        {
-            std::unique_lock<std::mutex> lk(mtx);
-            cv.wait(lk, [&] { return notified; });
-
-            if (connected) {
-                break;
-            } else {
-                notified = false;
-                lk.unlock();
-                std::this_thread::sleep_for(std::chrono::milliseconds(100));
-            }
-        }
-    }
-}
-
-auto Socket::Send(MessagePtr& msg, int /*timeout*/) -> int64_t
-{
-    // timeout argument not yet implemented
-
-    std::vector<MessagePtr> msgVec;
-    msgVec.reserve(1);
-    msgVec.emplace_back(std::move(msg));
-
-    return Send(msgVec);
-}
-
-auto Socket::Send(std::vector<MessagePtr>& msgVec, int /*timeout*/) -> int64_t
-try {
-    // timeout argument not yet implemented
-
-    int size(0);
-    for (auto& msg : msgVec) {
-        size += msg->GetSize();
-    }
-
-    fSendPushSem.wait();
-    {
-        std::lock_guard<std::mutex> lk(fSendQueueMutex);
-        fSendQueue.emplace(std::move(msgVec));
-    }
-    fSendPopSem.signal();
-
-    return size;
-} catch (const std::exception& e) {
-    LOG(error) << e.what();
-    return static_cast<int64_t>(TransferCode::error);
-}
-
-auto Socket::SendQueueReader() -> void
-{
-    fSendPopSem.async_wait([&] {
-        // Read msg from send queue
-        std::unique_lock<std::mutex> lk(fSendQueueMutex);
-        std::vector<MessagePtr> msgVec(std::move(fSendQueue.front()));
-        fSendQueue.pop();
-        lk.unlock();
-
-        bool postMultiPartStartBuffer = msgVec.size() > 1;
-        for (auto& msg : msgVec) {
-            // Create control message
-            ofi::unique_ptr<ControlMessage> ctrl(nullptr);
-            if (postMultiPartStartBuffer) {
-                postMultiPartStartBuffer = false;
-                ctrl = MakeControlMessageWithPmr<PostMultiPartStartBuffer>(fControlMemPool);
-                ctrl->msg.postMultiPartStartBuffer.numParts = msgVec.size();
-                ctrl->msg.postMultiPartStartBuffer.size = msg->GetSize();
-            } else {
-                ctrl = MakeControlMessageWithPmr<PostBuffer>(fControlMemPool);
-                ctrl->msg.postBuffer.size = msg->GetSize();
-            }
-
-            // Send control message
-            asio::mutable_buffer ctrlMsg(ctrl.get(), sizeof(ControlMessage));
-
-            if (fNeedOfiMemoryRegistration) {
-                asiofi::memory_region mr(*fOfiDomain, ctrlMsg, asiofi::mr::access::send);
-                auto desc = mr.desc();
-                fControlEndpoint->send(ctrlMsg,
-                                       desc,
-                                       [&, ctrl2 = std::move(ctrlMsg), mr2 = std::move(mr)](
-                                           asio::mutable_buffer) mutable {});
-            } else {
-                fControlEndpoint->send(
-                    ctrlMsg, [&, ctrl2 = std::move(ctrl)](asio::mutable_buffer) mutable {});
-            }
-
-            // Send data message
-            const auto size = msg->GetSize();
-
-            if (size) {
-                asio::mutable_buffer buffer(msg->GetData(), size);
-
-                if (fNeedOfiMemoryRegistration) {
-                    asiofi::memory_region mr(*fOfiDomain, buffer, asiofi::mr::access::send);
-                    auto desc = mr.desc();
-
-                    fDataEndpoint->send(buffer,
-                                        desc,
-                                        [&, size, msg2 = std::move(msg), mr2 = std::move(mr)](
-                                            asio::mutable_buffer) mutable {
-                                            fBytesTx += size;
-                                            fMessagesTx++;
-                                            fSendPushSem.signal();
-                                        });
-                } else {
-                    fDataEndpoint->send(
-                        buffer, [&, size, msg2 = std::move(msg)](asio::mutable_buffer) mutable {
-                            fBytesTx += size;
-                            fMessagesTx++;
-                            fSendPushSem.signal();
-                        });
-                }
-            } else {
-                ++fMessagesTx;
-                fSendPushSem.signal();
-            }
-        }
-
-        asio::dispatch(fContext.GetIoContext(), std::bind(&Socket::SendQueueReader, this));
-    });
-}
-
-auto Socket::SendQueueReaderStatic() -> void
-{
-    fSendPopSem.async_wait([&] {
-        // Read msg from send queue
-        std::unique_lock<std::mutex> lk(fSendQueueMutex);
-        std::vector<MessagePtr> msgVec(std::move(fSendQueue.front()));
-        fSendQueue.pop();
-        lk.unlock();
-
-        bool postMultiPartStartBuffer = msgVec.size() > 1;
-        if (postMultiPartStartBuffer) {
-            throw SocketError{tools::ToString("Multipart API not supported in static size mode.")};
-        }
-
-        MessagePtr& msg = msgVec[0];
-
-        // Send data message
-        const auto size = msg->GetSize();
-
-        if (size) {
-            asio::mutable_buffer buffer(msg->GetData(), size);
-
-            if (fNeedOfiMemoryRegistration) {
-                asiofi::memory_region mr(*fOfiDomain, buffer, asiofi::mr::access::send);
-                auto desc = mr.desc();
-
-                fDataEndpoint->send(buffer,
-                                    desc,
-                                    [&, size, msg2 = std::move(msg), mr2 = std::move(mr)](
-                                        asio::mutable_buffer) mutable {
-                                        fBytesTx += size;
-                                        fMessagesTx++;
-                                        fSendPushSem.signal();
-                                    });
-            } else {
-                fDataEndpoint->send(
-                    buffer, [&, size, msg2 = std::move(msg)](asio::mutable_buffer) mutable {
-                        fBytesTx += size;
-                        fMessagesTx++;
-                        fSendPushSem.signal();
-                    });
-            }
-        } else {
-            ++fMessagesTx;
-            fSendPushSem.signal();
-        }
-
-        asio::dispatch(fContext.GetIoContext(), std::bind(&Socket::SendQueueReaderStatic, this));
-    });
-}
-
-auto Socket::Receive(MessagePtr& msg, int /*timeout*/) -> int64_t
-try {
-    // timeout argument not yet implemented
-
-    fRecvPopSem.wait();
-    {
-        std::lock_guard<std::mutex> lk(fRecvQueueMutex);
-        msg = std::move(fRecvQueue.front().front());
-        fRecvQueue.pop();
-    }
-    fRecvPushSem.signal();
-
-    int size(msg->GetSize());
-    fBytesRx += size;
-    ++fMessagesRx;
-
-    return size;
-} catch (const std::exception& e) {
-    LOG(error) << e.what();
-    return static_cast<int>(TransferCode::error);
-}
-
-auto Socket::Receive(std::vector<MessagePtr>& msgVec, int /*timeout*/) -> int64_t
-try {
-    // timeout argument not yet implemented
-
-    fRecvPopSem.wait();
-    {
-        std::lock_guard<std::mutex> lk(fRecvQueueMutex);
-        msgVec = std::move(fRecvQueue.front());
-        fRecvQueue.pop();
-    }
-    fRecvPushSem.signal();
-
-    int64_t size(0);
-    for (auto& msg : msgVec) {
-        size += msg->GetSize();
-    }
-    fBytesRx += size;
-    ++fMessagesRx;
-
-    return size;
-} catch (const std::exception& e) {
-    LOG(error) << e.what();
-    return static_cast<int64_t>(TransferCode::error);
-}
-
-auto Socket::RecvControlQueueReader() -> void
-{
-    fRecvPushSem.async_wait([&] {
-        // Receive control message
-        ofi::unique_ptr<ControlMessage> ctrl(MakeControlMessageWithPmr<Empty>(fControlMemPool));
-        asio::mutable_buffer ctrlMsg(ctrl.get(), sizeof(ControlMessage));
-
-        if (fNeedOfiMemoryRegistration) {
-            asiofi::memory_region mr(*fOfiDomain, ctrlMsg, asiofi::mr::access::recv);
-            auto desc = mr.desc();
-
-            fControlEndpoint->recv(
-                ctrlMsg,
-                desc,
-                [&, ctrl2 = std::move(ctrl), mr2 = std::move(mr)](
-                    asio::mutable_buffer) mutable { OnRecvControl(std::move(ctrl2)); });
-        } else {
-            fControlEndpoint->recv(
-                ctrlMsg, [&, ctrl2 = std::move(ctrl)](asio::mutable_buffer) mutable {
-                    OnRecvControl(std::move(ctrl2));
-                });
-        }
-    });
-}
-
-auto Socket::OnRecvControl(ofi::unique_ptr<ControlMessage> ctrl) -> void
-{
-    // Check control message type
-    auto size(0);
-    if (ctrl->type == ControlMessageType::PostMultiPartStartBuffer) {
-        size = ctrl->msg.postMultiPartStartBuffer.size;
-        if (fMultiPartRecvCounter == -1) {
-            fMultiPartRecvCounter = ctrl->msg.postMultiPartStartBuffer.numParts;
-            assert(fInflightMultiPartMessage.empty());
-            fInflightMultiPartMessage.reserve(ctrl->msg.postMultiPartStartBuffer.numParts);
-        } else {
-            throw SocketError{tools::ToString(
-                "OFI transport: Received control start of new multi part message without completed "
-                "reception of previous multi part message. Number of parts missing: ",
-                fMultiPartRecvCounter)};
-        }
-    } else if (ctrl->type == ControlMessageType::PostBuffer) {
-        size = ctrl->msg.postBuffer.size;
-    } else {
-        throw SocketError{tools::ToString("OFI transport: Unknown control message type: '",
-                                          static_cast<int>(ctrl->type))};
-    }
-
-    // Receive data
-    auto msg = fContext.MakeReceiveMessage(size);
-
-    if (size) {
-        asio::mutable_buffer buffer(msg->GetData(), size);
-
-        if (fNeedOfiMemoryRegistration) {
-            asiofi::memory_region mr(*fOfiDomain, buffer, asiofi::mr::access::recv);
-            auto desc = mr.desc();
-
-            fDataEndpoint->recv(
-                buffer,
-                desc,
-                [&, msg2 = std::move(msg), mr2 = std::move(mr)](
-                    asio::mutable_buffer) mutable { DataMessageReceived(std::move(msg2)); });
-
-        } else {
-            fDataEndpoint->recv(buffer,
-                                [&, msg2 = std::move(msg)](asio::mutable_buffer) mutable {
-                                    DataMessageReceived(std::move(msg2));
-                                });
-        }
-    } else {
-        DataMessageReceived(std::move(msg));
-    }
-
-    asio::dispatch(fContext.GetIoContext(),
-                          std::bind(&Socket::RecvControlQueueReader, this));
-}
-
-auto Socket::RecvQueueReaderStatic() -> void
-{
-    fRecvPushSem.async_wait([&] {
-        static size_t size = fContext.GetSizeHint();
-        // Receive data
-        auto msg = fContext.MakeReceiveMessage(size);
-
-        if (size) {
-            asio::mutable_buffer buffer(msg->GetData(), size);
-
-            if (fNeedOfiMemoryRegistration) {
-                asiofi::memory_region mr(*fOfiDomain, buffer, asiofi::mr::access::recv);
-                auto desc = mr.desc();
-
-                fDataEndpoint->recv(buffer,
-                                    desc,
-                                    [&, msg2 = std::move(msg), mr2 = std::move(mr)](
-                                        asio::mutable_buffer) mutable {
-                                        DataMessageReceived(std::move(msg2));
-                                    });
-
-            } else {
-                fDataEndpoint->recv(
-                    buffer, [&, msg2 = std::move(msg)](asio::mutable_buffer) mutable {
-                        DataMessageReceived(std::move(msg2));
-                    });
-            }
-        } else {
-            DataMessageReceived(std::move(msg));
-        }
-
-        asio::dispatch(fContext.GetIoContext(),
-                              std::bind(&Socket::RecvQueueReaderStatic, this));
-    });
-}
-
-auto Socket::DataMessageReceived(MessagePtr msg) -> void
-{
-    if (fMultiPartRecvCounter > 0) {
-        --fMultiPartRecvCounter;
-        fInflightMultiPartMessage.push_back(std::move(msg));
-    }
-
-    if (fMultiPartRecvCounter == 0) {
-        std::unique_lock<std::mutex> lk(fRecvQueueMutex);
-        fRecvQueue.push(std::move(fInflightMultiPartMessage));
-        lk.unlock();
-        fMultiPartRecvCounter = -1;
-        fRecvPopSem.signal();
-    } else if (fMultiPartRecvCounter == -1) {
-        std::vector<MessagePtr> msgVec;
-        msgVec.push_back(std::move(msg));
-        std::unique_lock<std::mutex> lk(fRecvQueueMutex);
-        fRecvQueue.push(std::move(msgVec));
-        lk.unlock();
-        fRecvPopSem.signal();
-    }
-
-}
-
-auto Socket::Close() -> void {}
-
-auto Socket::SetOption(const string& /*option*/, const void* /*value*/, size_t /*valueSize*/) -> void
-{
-    // if (zmq_setsockopt(fControlSocket, GetConstant(option), value, valueSize) < 0) {
-        // throw SocketError{tools::ToString("Failed setting socket option, reason: ", zmq_strerror(errno))};
-    // }
-}
-
-auto Socket::GetOption(const string& /*option*/, void* /*value*/, size_t* /*valueSize*/) -> void
-{
-    // if (zmq_getsockopt(fControlSocket, GetConstant(option), value, valueSize) < 0) {
-        // throw SocketError{tools::ToString("Failed getting socket option, reason: ", zmq_strerror(errno))};
-    // }
-}
-
-void Socket::SetLinger(int /*value*/)
-{
-    LOG(debug) << "OFI transport (" << fId << "): Not yet implemented.";
-}
-
-int Socket::GetLinger() const
-{
-    LOG(debug) << "OFI transport (" << fId << "): Not yet implemented.";
-    return 0;
-}
-
-void Socket::SetSndBufSize(int /*value*/)
-{
-    LOG(debug) << "OFI transport (" << fId << "): Not yet implemented.";
-}
-
-int Socket::GetSndBufSize() const
-{
-    LOG(debug) << "OFI transport (" << fId << "): Not yet implemented.";
-    return 0;
-}
-
-void Socket::SetRcvBufSize(int /*value*/)
-{
-    LOG(debug) << "OFI transport (" << fId << "): Not yet implemented.";
-}
-
-int Socket::GetRcvBufSize() const
-{
-    LOG(debug) << "OFI transport (" << fId << "): Not yet implemented.";
-    return 0;
-}
-
-void Socket::SetSndKernelSize(int /*value*/)
-{
-    LOG(debug) << "OFI transport (" << fId << "): Not yet implemented.";
-}
-
-int Socket::GetSndKernelSize() const
-{
-    LOG(debug) << "OFI transport (" << fId << "): Not yet implemented.";
-    return 0;
-}
-
-void Socket::SetRcvKernelSize(int /*value*/)
-{
-    LOG(debug) << "OFI transport (" << fId << "): Not yet implemented.";
-}
-
-int Socket::GetRcvKernelSize() const
-{
-    LOG(debug) << "OFI transport (" << fId << "): Not yet implemented.";
-    return 0;
-}
-
-auto Socket::GetConstant(const string& /*constant*/) -> int
-{
-    LOG(debug) << "OFI transport: Not yet implemented.";
-    return -1;
-}
-
-Socket::~Socket()
-{
-    try {
-        Close(); // NOLINT(clang-analyzer-optin.cplusplus.VirtualCall)
-    } catch (SocketError& e) {
-        LOG(error) << e.what();
-    }
-}
-
-} // namespace fair::mq::ofi

fairmq/ofi/Socket.h

@@ -1,125 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2018-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef FAIR_MQ_OFI_SOCKET_H
-#define FAIR_MQ_OFI_SOCKET_H
-
-#include <fairmq/Message.h>
-#include <fairmq/Socket.h>
-#include <fairmq/ofi/Context.h>
-#include <fairmq/ofi/ControlMessages.h>
-
-#include <asiofi/connected_endpoint.hpp>
-#include <asiofi/memory_resources.hpp>
-#include <asiofi/passive_endpoint.hpp>
-#include <asiofi/semaphore.hpp>
-#include <memory> // unique_ptr
-#include <mutex>
-
-
-namespace fair::mq::ofi
-{
-
-/**
- * @class Socket Socket.h <fairmq/ofi/Socket.h>
- * @brief
- *
- * @todo TODO insert long description
- */
-class Socket final : public fair::mq::Socket
-{
-  public:
-    Socket(Context& context, const std::string& type, const std::string& name, const std::string& id = "");
-    Socket(const Socket&) = delete;
-    Socket(Socket&&) = delete;
-    Socket& operator=(const Socket&) = delete;
-    Socket& operator=(Socket&&) = delete;
-
-    auto GetId() const -> std::string override { return fId; }
-
-    auto Events(uint32_t *events) -> int override { *events = 0; return -1; }
-    auto Bind(const std::string& address) -> bool override;
-    auto Connect(const std::string& address) -> bool override;
-
-    auto Send(MessagePtr& msg, int timeout = 0) -> int64_t override;
-    auto Receive(MessagePtr& msg, int timeout = 0) -> int64_t override;
-    auto Send(std::vector<MessagePtr>& msgVec, int timeout = 0) -> int64_t override;
-    auto Receive(std::vector<MessagePtr>& msgVec, int timeout = 0) -> int64_t override;
-
-    auto GetSocket() const -> void* { return nullptr; }
-
-    void SetLinger(int value) override;
-    int GetLinger() const override;
-    void SetSndBufSize(int value) override;
-    int GetSndBufSize() const override;
-    void SetRcvBufSize(int value) override;
-    int GetRcvBufSize() const override;
-    void SetSndKernelSize(int value) override;
-    int GetSndKernelSize() const override;
-    void SetRcvKernelSize(int value) override;
-    int GetRcvKernelSize() const override;
-
-    auto Close() -> void override;
-
-    auto SetOption(const std::string& option, const void* value, size_t valueSize) -> void override;
-    auto GetOption(const std::string& option, void* value, size_t* valueSize) -> void override;
-
-    auto GetBytesTx() const -> unsigned long override { return fBytesTx; }
-    auto GetBytesRx() const -> unsigned long override { return fBytesRx; }
-    auto GetMessagesTx() const -> unsigned long override { return fMessagesTx; }
-    auto GetMessagesRx() const -> unsigned long override { return fMessagesRx; }
-
-    auto GetNumberOfConnectedPeers() const -> unsigned long override
-    {
-        throw SocketError("not yet implemented");
-    }
-
-    static auto GetConstant(const std::string& constant) -> int;
-
-    ~Socket() override;
-
-  private:
-    Context& fContext;
-    asiofi::allocated_pool_resource fControlMemPool;
-    std::unique_ptr<asiofi::info> fOfiInfo;
-    std::unique_ptr<asiofi::fabric> fOfiFabric;
-    std::unique_ptr<asiofi::domain> fOfiDomain;
-    std::unique_ptr<asiofi::passive_endpoint> fPassiveEndpoint;
-    std::unique_ptr<asiofi::connected_endpoint> fDataEndpoint, fControlEndpoint;
-    std::string fId;
-    std::atomic<unsigned long> fBytesTx;
-    std::atomic<unsigned long> fBytesRx;
-    std::atomic<unsigned long> fMessagesTx;
-    std::atomic<unsigned long> fMessagesRx;
-    Address fRemoteAddr;
-    Address fLocalAddr;
-    std::mutex fSendQueueMutex, fRecvQueueMutex;
-    std::queue<std::vector<MessagePtr>> fSendQueue, fRecvQueue;
-    std::vector<MessagePtr> fInflightMultiPartMessage;
-    int64_t fMultiPartRecvCounter;
-    asiofi::synchronized_semaphore fSendPushSem, fSendPopSem, fRecvPushSem, fRecvPopSem;
-    std::atomic<bool> fNeedOfiMemoryRegistration;
-
-    auto InitOfi(Address addr) -> void;
-    auto BindControlEndpoint() -> void;
-    auto BindDataEndpoint() -> void;
-    enum class Band { Control, Data };
-    auto ConnectEndpoint(std::unique_ptr<asiofi::connected_endpoint>& endpoint, Band type) -> void;
-    auto SendQueueReader() -> void;
-    auto SendQueueReaderStatic() -> void;
-    auto RecvControlQueueReader() -> void;
-    auto RecvQueueReaderStatic() -> void;
-    auto OnRecvControl(ofi::unique_ptr<ControlMessage> ctrl) -> void;
-    auto DataMessageReceived(MessagePtr msg) -> void;
-}; /* class Socket */
-
-struct SilentSocketError : SocketError { using SocketError::SocketError; };
-
-} // namespace fair::mq::ofi
-
-#endif /* FAIR_MQ_OFI_SOCKET_H */

fairmq/ofi/TransportFactory.h

@@ -1,218 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2018-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef FAIR_MQ_OFI_TRANSPORTFACTORY_H
-#define FAIR_MQ_OFI_TRANSPORTFACTORY_H
-
-#include <asiofi.hpp>
-#include <cstddef>
-#include <cstdint>
-#include <fairlogger/Logger.h>
-#include <fairmq/Channel.h>
-#include <fairmq/Message.h>
-#include <fairmq/Poller.h>
-#include <fairmq/ProgOptions.h>
-#include <fairmq/Socket.h>
-#include <fairmq/TransportFactory.h>
-#include <fairmq/Transports.h>
-#include <fairmq/ofi/Context.h>
-#include <fairmq/ofi/Message.h>
-#include <fairmq/ofi/Socket.h>
-#include <memory>
-#include <stdexcept>
-#include <string>
-#include <unordered_map>
-#include <vector>
-
-namespace fair::mq::ofi {
-
-/**
- * @class TransportFactory TransportFactory.h <fairmq/ofi/TransportFactory.h>
- * @brief FairMQ transport factory for the ofi transport
- *
- * @todo TODO insert long description
- */
-struct TransportFactory final : mq::TransportFactory
-{
-    TransportFactory(std::string const& id = "", ProgOptions const* config = nullptr)
-        : mq::TransportFactory(id)
-        , fContext(*this, *this, 1)
-    {
-        try {
-            LOG(debug) << "OFI transport: asiofi (" << fContext.GetAsiofiVersion() << ")";
-
-            if (config) {
-                fContext.SetSizeHint(config->GetProperty<size_t>("ofi-size-hint", 0));
-            }
-        } catch (ContextError& e) {
-            throw TransportFactoryError(e.what());
-        }
-    }
-
-    TransportFactory(const TransportFactory&) = delete;
-    TransportFactory(TransportFactory&&) = delete;
-    TransportFactory& operator=(const TransportFactory&) = delete;
-    TransportFactory& operator=(TransportFactory&&) = delete;
-    ~TransportFactory() override = default;
-
-    auto CreateMessage() -> std::unique_ptr<mq::Message> override
-    {
-        return std::make_unique<Message>(&fMemoryResource);
-    }
-
-    auto CreateMessage(Alignment /*alignment*/) -> std::unique_ptr<mq::Message> override
-    {
-        // TODO Do not ignore alignment
-        return std::make_unique<Message>(&fMemoryResource);
-    }
-
-    auto CreateMessage(std::size_t size) -> std::unique_ptr<mq::Message> override
-    {
-        return std::make_unique<Message>(&fMemoryResource, size);
-    }
-
-    auto CreateMessage(std::size_t size, Alignment /*alignment*/)
-        -> std::unique_ptr<mq::Message> override
-    {
-        // TODO Do not ignore alignment
-        return std::make_unique<Message>(&fMemoryResource, size);
-    }
-
-    auto CreateMessage(void* data, std::size_t size, FreeFn* ffn, void* hint = nullptr)
-        -> std::unique_ptr<mq::Message> override
-    {
-        return std::make_unique<Message>(&fMemoryResource, data, size, ffn, hint);
-    }
-
-    auto CreateMessage(std::unique_ptr<mq::UnmanagedRegion>& region,
-                       void* data,
-                       std::size_t size,
-                       void* hint = nullptr) -> std::unique_ptr<mq::Message> override
-    {
-        return std::make_unique<Message>(&fMemoryResource, region, data, size, hint);
-    }
-
-    auto CreateSocket(std::string const& type, std::string const& name)
-        -> std::unique_ptr<mq::Socket> override
-    {
-        return std::make_unique<Socket>(fContext, type, name, GetId());
-    }
-
-    auto CreatePoller(std::vector<mq::Channel> const& /*channels*/) const
-        -> std::unique_ptr<mq::Poller> override
-    {
-        throw std::runtime_error("Not yet implemented (Poller).");
-    }
-
-    auto CreatePoller(std::vector<mq::Channel*> const& /*channels*/) const
-        -> std::unique_ptr<mq::Poller> override
-    {
-        throw std::runtime_error("Not yet implemented (Poller).");
-    }
-
-    auto CreatePoller(
-        std::unordered_map<std::string, std::vector<Channel>> const& /*channelsMap*/,
-        std::vector<std::string> const& /*channelList*/) const
-        -> std::unique_ptr<mq::Poller> override
-    {
-        throw std::runtime_error("Not yet implemented (Poller).");
-    }
-
-    auto CreateUnmanagedRegion(std::size_t /*size*/,
-                               RegionCallback /*callback = nullptr*/,
-                               std::string const& /*path = ""*/,
-                               int /*flags = 0*/,
-                               RegionConfig /*cfg = RegionConfig()*/)
-        -> std::unique_ptr<mq::UnmanagedRegion> override
-    {
-        throw std::runtime_error("Not yet implemented UMR.");
-    }
-
-    auto CreateUnmanagedRegion(std::size_t /*size*/,
-                               RegionBulkCallback /*callback = nullptr*/,
-                               std::string const& /*path = ""*/,
-                               int /*flags = 0*/,
-                               RegionConfig /*cfg = RegionConfig()*/)
-        -> std::unique_ptr<mq::UnmanagedRegion> override
-    {
-        throw std::runtime_error("Not yet implemented UMR.");
-    }
-
-    auto CreateUnmanagedRegion(std::size_t /*size*/,
-                               int64_t /*userFlags*/,
-                               RegionCallback /*callback = nullptr*/,
-                               std::string const& /*path = ""*/,
-                               int /*flags = 0*/,
-                               RegionConfig /*cfg = RegionConfig()*/)
-        -> std::unique_ptr<mq::UnmanagedRegion> override
-    {
-        throw std::runtime_error("Not yet implemented UMR.");
-    }
-
-    auto CreateUnmanagedRegion(std::size_t /*size*/,
-                               int64_t /*userFlags*/,
-                               RegionBulkCallback /*callback = nullptr*/,
-                               std::string const& /*path = ""*/,
-                               int /*flags = 0*/,
-                               RegionConfig /*cfg = RegionConfig()*/)
-        -> std::unique_ptr<mq::UnmanagedRegion> override
-    {
-        throw std::runtime_error("Not yet implemented UMR.");
-    }
-
-    auto CreateUnmanagedRegion(std::size_t /*size*/,
-                               RegionCallback /*callback*/,
-                               RegionConfig /*cfg*/)
-        -> std::unique_ptr<mq::UnmanagedRegion> override
-    {
-        throw std::runtime_error("Not yet implemented UMR.");
-    }
-
-    auto CreateUnmanagedRegion(std::size_t /*size*/,
-                               RegionBulkCallback /*callback*/,
-                               RegionConfig /*cfg*/)
-        -> std::unique_ptr<mq::UnmanagedRegion> override
-    {
-        throw std::runtime_error("Not yet implemented UMR.");
-    }
-
-    auto SubscribeToRegionEvents(RegionEventCallback /*callback*/) -> void override
-    {
-        throw std::runtime_error("Not yet implemented.");
-    }
-
-    auto SubscribedToRegionEvents() -> bool override
-    {
-        throw std::runtime_error("Not yet implemented.");
-    }
-
-    auto UnsubscribeFromRegionEvents() -> void override
-    {
-        throw std::runtime_error("Not yet implemented.");
-    }
-
-    auto GetRegionInfo() -> std::vector<RegionInfo> override
-    {
-        LOG(error) << "GetRegionInfo not yet implemented for OFI, returning empty vector";
-        return std::vector<RegionInfo>();
-    }
-
-    auto GetType() const -> Transport override { return Transport::OFI; }
-
-    void Interrupt() override { fContext.Interrupt(); }
-    void Resume() override { fContext.Resume(); }
-    void Reset() override { fContext.Reset(); }
-
-  private:
-    mutable Context fContext;
-    asiofi::allocated_pool_resource fMemoryResource;
-}; /* class TransportFactory */
-
-}   // namespace fair::mq::ofi
-
-#endif /* FAIR_MQ_OFI_TRANSPORTFACTORY_H */

fairmq/plugins/PMIx/CMakeLists.txt

@@ -1,33 +0,0 @@
-################################################################################
-# Copyright (C) 2019-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  #
-#                                                                              #
-#              This software is distributed under the terms of the             #
-#              GNU Lesser General Public Licence (LGPL) version 3,             #
-#                  copied verbatim in the file "LICENSE"                       #
-################################################################################
-
-set(plugin FairMQPlugin_pmix)
-add_library(${plugin} SHARED
-  ${CMAKE_CURRENT_SOURCE_DIR}/PMIxPlugin.cxx
-  ${CMAKE_CURRENT_SOURCE_DIR}/PMIxPlugin.h
-  ${CMAKE_CURRENT_SOURCE_DIR}/PMIxCommands.h
-  ${CMAKE_CURRENT_SOURCE_DIR}/PMIx.hpp
-)
-target_compile_features(${plugin} PUBLIC cxx_std_17)
-target_link_libraries(${plugin} PUBLIC FairMQ PMIx::libpmix)
-target_include_directories(${plugin} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
-set_target_properties(${plugin} PROPERTIES
-  CXX_VISIBILITY_PRESET hidden
-  OUTPUT_NAME "${PROJECT_NAME_LOWER}-plugin-pmix"
-)
-
-set(exe fairmq-pmix-command-ui)
-add_executable(${exe} ${CMAKE_CURRENT_SOURCE_DIR}/runPMIxCommandUI.cxx)
-target_link_libraries(${exe} FairMQ PMIx::libpmix)
-target_include_directories(${exe} PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
-
-install(TARGETS ${plugin} ${exe}
-  EXPORT ${PROJECT_EXPORT_SET}
-  LIBRARY DESTINATION ${PROJECT_INSTALL_LIBDIR}
-  RUNTIME DESTINATION ${PROJECT_INSTALL_BINDIR}
-)

fairmq/plugins/PMIx/PMIx.hpp

@@ -1,317 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef PMIX_HPP
-#define PMIX_HPP
-
-#include <array>
-#include <cstring>
-#include <functional>
-#include <limits>
-#include <memory>
-#include <ostream>
-#include <pmix.h>
-#include <sstream>
-#include <stdexcept>
-#include <string>
-#include <type_traits>
-#include <utility>
-#include <vector>
-
-// C++ PMIx v2.2 API
-namespace pmix
-{
-
-struct runtime_error : std::runtime_error
-{
-    using std::runtime_error::runtime_error;
-};
-
-using status = pmix_status_t;
-
-using nspace = pmix_nspace_t;
-
-using key = pmix_key_t;
-
-using data_type = pmix_data_type_t;
-
-struct rank
-{
-    enum named : pmix_rank_t
-    {
-        undef = PMIX_RANK_UNDEF,
-        wildcard = PMIX_RANK_WILDCARD,
-        local_node = PMIX_RANK_LOCAL_NODE
-    };
-
-    explicit rank(pmix_rank_t r)
-        : m_value(r)
-    {}
-
-    operator pmix_rank_t() { return m_value; }
-
-  private:
-    pmix_rank_t m_value;
-};
-
-struct proc : pmix_proc_t
-{
-    proc() { PMIX_PROC_CONSTRUCT(static_cast<pmix_proc_t*>(this)); }
-    ~proc() { PMIX_PROC_DESTRUCT(static_cast<pmix_proc_t*>(this)); }
-
-    proc(pmix::nspace ns, pmix::rank r)
-    {
-        PMIX_PROC_LOAD(static_cast<pmix_proc_t*>(this), ns, static_cast<pmix_rank_t>(r));
-    }
-
-    friend std::ostream& operator<<(std::ostream& os, const proc& p)
-    {
-        return os << p.nspace << "_" << p.rank;
-    }
-};
-
-struct value : pmix_value_t
-{
-    value() { PMIX_VALUE_CONSTRUCT(static_cast<pmix_value_t*>(this)); }
-    ~value() { PMIX_VALUE_DESTRUCT(static_cast<pmix_value_t*>(this)); }
-
-    value(const value& rhs)
-    {
-        status rc;
-        auto lhs(static_cast<pmix_value_t*>(this));
-        PMIX_VALUE_XFER(rc, lhs, static_cast<pmix_value_t*>(const_cast<value*>(&rhs)));
-
-        if (rc != PMIX_SUCCESS) {
-            throw runtime_error("pmix::value copy ctor failed: rc=" + std::to_string(rc));
-        }
-    }
-
-    template<typename T>
-    explicit value(T)
-    {
-        throw runtime_error("Given value type not supported or not yet implemented.");
-    }
-
-    explicit value(const char* val)
-    {
-        PMIX_VALUE_LOAD(static_cast<pmix_value_t*>(this), const_cast<char*>(val), PMIX_STRING);
-    }
-
-    explicit value(const std::string& val)
-    {
-        PMIX_VALUE_LOAD(
-            static_cast<pmix_value_t*>(this), const_cast<char*>(val.c_str()), PMIX_STRING);
-    }
-
-    explicit value(int val)
-    {
-        PMIX_VALUE_LOAD(static_cast<pmix_value_t*>(this), &val, PMIX_INT);
-    }
-
-    explicit value(pmix_data_array_t* val)
-    {
-        PMIX_VALUE_LOAD(static_cast<pmix_value_t*>(this), val, PMIX_DATA_ARRAY);
-    }
-};
-
-struct info : pmix_info_t
-{
-    info() { PMIX_INFO_CONSTRUCT(static_cast<pmix_info_t*>(this)); }
-    ~info() { PMIX_INFO_DESTRUCT(static_cast<pmix_info_t*>(this)); }
-
-    template<typename... Args>
-    info(const std::string& k, Args&&... args)
-    {
-        (void)strncpy(key, k.c_str(), PMIX_MAX_KEYLEN);
-        flags = 0;
-
-        pmix::value rhs(std::forward<Args>(args)...);
-        auto lhs(&value);
-        status rc;
-        PMIX_VALUE_XFER(rc, lhs, static_cast<pmix_value_t*>(&rhs));
-
-        if (rc != PMIX_SUCCESS) {
-            throw runtime_error("pmix::info ctor failed: rc=" + std::to_string(rc));
-        }
-    }
-
-    friend std::ostream& operator<<(std::ostream& os, const info& i)
-    {
-        return os << "key=" << i.key << ",value='" << i.value.data.string << "'";
-    }
-
-    info(const info& rhs)
-    {
-        PMIX_INFO_XFER(static_cast<pmix_info_t*>(this),
-                       static_cast<pmix_info_t*>(const_cast<info*>(&rhs)));
-    }
-};
-
-struct pdata : pmix_pdata_t
-{
-    pdata() { PMIX_PDATA_CONSTRUCT(static_cast<pmix_pdata_t*>(this)); }
-    ~pdata() { PMIX_PDATA_DESTRUCT(static_cast<pmix_pdata_t*>(this)); }
-
-    pdata(const pdata& rhs)
-    {
-        PMIX_PDATA_XFER(static_cast<pmix_pdata_t*>(this),
-                        static_cast<pmix_pdata_t*>(const_cast<pdata*>(&rhs)));
-    }
-
-    auto set_key(const std::string& new_key) -> void
-    {
-        (void)strncpy(key, new_key.c_str(), PMIX_MAX_KEYLEN);
-    }
-};
-
-auto init(const std::vector<info>& info = {}) -> proc
-{
-    proc res;
-    status rc;
-
-    rc = PMIx_Init(&res, const_cast<pmix::info*>(info.data()), info.size());
-    if (rc != PMIX_SUCCESS) {
-        throw runtime_error("pmix::init() failed: rc=" + std::to_string(rc));
-    }
-
-    return res;
-}
-
-auto initialized() -> bool { return !!PMIx_Initialized(); }
-
-auto get_version() -> std::string { return {PMIx_Get_version()}; }
-
-auto finalize(const std::vector<info>& info = {}) -> void
-{
-    status rc;
-
-    rc = PMIx_Finalize(info.data(), info.size());
-    if (rc != PMIX_SUCCESS) {
-        throw runtime_error("pmix::finalize() failed: rc=" + std::to_string(rc));
-    }
-}
-
-auto publish(const std::vector<info>& info) -> void
-{
-    status rc;
-
-    rc = PMIx_Publish(info.data(), info.size());
-    if (rc != PMIX_SUCCESS) {
-        throw runtime_error("pmix::publish() failed: rc=" + std::to_string(rc));
-    }
-}
-
-auto fence(const std::vector<proc>& procs = {}, const std::vector<info>& info = {}) -> void
-{
-    status rc;
-
-    rc = PMIx_Fence(procs.data(), procs.size(), info.data(), info.size());
-    if (rc != PMIX_SUCCESS) {
-        throw runtime_error("pmix::fence() failed: rc=" + std::to_string(rc));
-    }
-}
-
-auto lookup(std::vector<pdata>& pdata, const std::vector<info>& info = {}) -> void
-{
-    status rc;
-
-    rc = PMIx_Lookup(pdata.data(), pdata.size(), info.data(), info.size());
-    if (rc != PMIX_SUCCESS) {
-        throw runtime_error("pmix::lookup() failed: rc=" + std::to_string(rc));
-    }
-}
-
-std::string get_info(const std::string& name, pmix::proc& process)
-{
-    pmix_value_t* v;
-
-    pmix::status rc = PMIx_Get(&process, name.c_str(), nullptr, 0, &v);
-    if (rc == PMIX_SUCCESS) {
-        std::stringstream ss;
-
-        switch (v->type) {
-            case PMIX_SIZE:      ss << static_cast<size_t>(v->data.size)       << " (size_t)";       break;
-            case PMIX_INT:       ss << static_cast<int>(v->data.integer)       << " (int)";          break;
-            case PMIX_INT8:      ss << static_cast<int8_t>(v->data.int8)       << " (int8_t)";       break;
-            case PMIX_INT16:     ss << static_cast<int16_t>(v->data.int16)     << " (int16_t)";      break;
-            case PMIX_INT32:     ss << static_cast<int32_t>(v->data.int32)     << " (int32_t)";      break;
-            case PMIX_INT64:     ss << static_cast<int64_t>(v->data.int64)     << " (int64_t)";      break;
-            case PMIX_UINT:      ss << static_cast<unsigned int>(v->data.uint) << " (unsigned int)"; break;
-            case PMIX_UINT8:     ss << static_cast<uint8_t>(v->data.uint8)     << " (uint8_t)";      break;
-            case PMIX_UINT16:    ss << static_cast<uint16_t>(v->data.uint16)   << " (uint16_t)";     break;
-            case PMIX_UINT32:    ss << static_cast<uint32_t>(v->data.uint32)   << " (uint32_t)";     break;
-            case PMIX_UINT64:    ss << static_cast<uint64_t>(v->data.uint64)   << " (uint64_t)";     break;
-            case PMIX_FLOAT:     ss << static_cast<float>(v->data.fval)        << " (float)";        break;
-            case PMIX_DOUBLE:    ss << static_cast<double>(v->data.dval)       << " (double)";       break;
-            case PMIX_PID:       ss << static_cast<pid_t>(v->data.pid)         << " (pid_t)";        break;
-            case PMIX_STRING:    ss << static_cast<char*>(v->data.string)      << " (string)";       break;
-            case PMIX_PROC_RANK: ss << static_cast<uint32_t>(v->data.rank)     << " (pmix_rank_t)";  break;
-            case PMIX_PROC:      ss << "proc.nspace: " << static_cast<pmix_proc_t*>(v->data.proc)->nspace
-                                    << ", proc.rank: " << static_cast<pmix_proc_t*>(v->data.proc)->rank << " (pmix_proc_t*)"; break;
-            default:
-                ss << "unknown type: " << v->type;
-                break;
-        }
-
-        return ss.str();
-    } else if (rc == PMIX_ERR_NOT_FOUND) {
-        // LOG(error) << "PMIx_Get failed: PMIX_ERR_NOT_FOUND";
-        return "";
-    } else {
-        // LOG(error) << "PMIx_Get failed: " << rc;
-        return "<undefined>";
-    }
-}
-
-std::string get_value_str(const pmix_value_t& v)
-{
-    switch (v.type) {
-        case PMIX_BOOL:       return std::to_string(static_cast<bool>(v.data.flag));
-        case PMIX_SIZE:       return std::to_string(static_cast<size_t>(v.data.size));
-        case PMIX_INT:        return std::to_string(static_cast<int>(v.data.integer));
-        case PMIX_INT8:       return std::to_string(static_cast<int8_t>(v.data.int8));
-        case PMIX_INT16:      return std::to_string(static_cast<int16_t>(v.data.int16));
-        case PMIX_INT32:      return std::to_string(static_cast<int32_t>(v.data.int32));
-        case PMIX_INT64:      return std::to_string(static_cast<int64_t>(v.data.int64));
-        case PMIX_UINT:       return std::to_string(static_cast<unsigned int>(v.data.uint));
-        case PMIX_UINT8:      return std::to_string(static_cast<uint8_t>(v.data.uint8));
-        case PMIX_UINT16:     return std::to_string(static_cast<uint16_t>(v.data.uint16));
-        case PMIX_UINT32:     return std::to_string(static_cast<uint32_t>(v.data.uint32));
-        case PMIX_UINT64:     return std::to_string(static_cast<uint64_t>(v.data.uint64));
-        case PMIX_FLOAT:      return std::to_string(static_cast<float>(v.data.fval));
-        case PMIX_DOUBLE:     return std::to_string(static_cast<double>(v.data.dval));
-        case PMIX_PID:        return std::to_string(static_cast<pid_t>(v.data.pid));
-        case PMIX_STRING:     return static_cast<char*>(v.data.string);
-        case PMIX_PROC_RANK:  return std::to_string(static_cast<uint32_t>(v.data.rank));
-        case PMIX_POINTER:    { std::stringstream ss; ss << static_cast<void*>(v.data.ptr); return ss.str(); }
-        case PMIX_DATA_ARRAY: {
-            if (v.data.darray->type == PMIX_PROC) {
-                std::stringstream ss;
-                ss << "[";
-                for (size_t i = 0; i < v.data.darray->size; ++i) {
-                    ss << static_cast<pmix_proc_t*>(static_cast<pmix_data_array_t*>(v.data.darray)->array)[0].nspace;
-                    ss << "_";
-                    ss << static_cast<pmix_proc_t*>(static_cast<pmix_data_array_t*>(v.data.darray)->array)[0].rank;
-
-                    if (i < v.data.darray->size - 1) {
-                        ss << ",";
-                    }
-                }
-                ss << "]";
-                return ss.str();
-            } else {
-                return "UNKNOWN TYPE IN DATA ARRAY";
-            }
-        }
-        default:             return "UNKNOWN TYPE";
-    }
-}
-
-} /* namespace pmix */
-
-#endif /* PMIX_HPP */

fairmq/plugins/PMIx/PMIxCommands.h

@@ -1,291 +0,0 @@
-/********************************************************************************
- *    Copyright (C) 2019 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH    *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef PMIXCOMMANDS_H
-#define PMIXCOMMANDS_H
-
-#include "PMIx.hpp"
-
-#include <fairlogger/Logger.h>
-#include <fairmq/tools/Semaphore.h>
-#include <memory> // make_unique
-#include <string>
-
-namespace pmix
-{
-
-std::array<std::string, 47> typeNames =
-{
-    {
-        "PMIX_UNDEF",
-        "PMIX_BOOL",
-        "PMIX_BYTE",
-        "PMIX_STRING",
-        "PMIX_SIZE",
-        "PMIX_PID",
-        "PMIX_INT",
-        "PMIX_INT8",
-        "PMIX_INT16",
-        "PMIX_INT32",
-        "PMIX_INT64",
-        "PMIX_UINT",
-        "PMIX_UINT8",
-        "PMIX_UINT16",
-        "PMIX_UINT32",
-        "PMIX_UINT64",
-        "PMIX_FLOAT",
-        "PMIX_DOUBLE",
-        "PMIX_TIMEVAL",
-        "PMIX_TIME",
-        "PMIX_STATUS",
-        "PMIX_VALUE",
-        "PMIX_PROC",
-        "PMIX_APP",
-        "PMIX_INFO",
-        "PMIX_PDATA",
-        "PMIX_BUFFER",
-        "PMIX_BYTE_OBJECT",
-        "PMIX_KVAL",
-        "PMIX_MODEX",
-        "PMIX_PERSIST",
-        "PMIX_POINTER",
-        "PMIX_SCOPE",
-        "PMIX_DATA_RANGE",
-        "PMIX_COMMAND",
-        "PMIX_INFO_DIRECTIVES",
-        "PMIX_DATA_TYPE",
-        "PMIX_PROC_STATE",
-        "PMIX_PROC_INFO",
-        "PMIX_DATA_ARRAY",
-        "PMIX_PROC_RANK",
-        "PMIX_QUERY",
-        "PMIX_COMPRESSED_STRING",
-        "PMIX_ALLOC_DIRECTIVE",
-        "PMIX_INFO_ARRAY",
-        "PMIX_IOF_CHANNEL",
-        "PMIX_ENVAR"
-    }
-};
-
-enum class Command : int
-{
-    general = PMIX_EXTERNAL_ERR_BASE,
-    error = PMIX_EXTERNAL_ERR_BASE - 1
-};
-
-
-class Commands
-{
-  public:
-    Commands(const proc& process)
-        : fProcess(process)
-        , fSubscribed(false)
-    {
-    }
-
-    ~Commands()
-    {
-        Unsubscribe();
-    }
-
-    void Subscribe(std::function<void(const std::string& msg, const proc& sender)> callback)
-    {
-        using namespace std::placeholders;
-
-        LOG(debug) << "PMIxCommands: Subscribing...";
-
-        fCallback = callback;
-        std::array<pmix::status, 1> codes;
-        codes[0] = static_cast<int>(pmix::Command::general);
-
-        PMIX_INFO_LOAD(&(fInfos[0]), PMIX_EVENT_RETURN_OBJECT, this, PMIX_POINTER);
-
-        PMIx_Register_event_handler(codes.data(), codes.size(),
-                                    fInfos.data(), fInfos.size(),
-                                    &Commands::Handler,
-                                    &Commands::EventHandlerRegistration,
-                                    this);
-        fBlocker.Wait();
-        LOG(debug) << "PMIxCommands: Subscribing complete!";
-    }
-
-    void Unsubscribe()
-    {
-        if (fSubscribed) {
-            LOG(debug) << "PMIxCommands: Unsubscribing...";
-            PMIx_Deregister_event_handler(fHandlerRef, &Commands::EventHandlerDeregistration, this);
-            fBlocker.Wait();
-            LOG(debug) << "PMIxCommands: Unsubscribing complete!";
-        } else {
-            LOG(debug) << "Unsubscribe() is called while no subscription is active";
-        }
-    }
-
-    struct Holder
-    {
-        Holder() : fData(nullptr) {}
-        ~Holder() { PMIX_DATA_ARRAY_FREE(fData); }
-
-        std::vector<pmix::info> fInfos;
-        pmix_data_array_t* fData;
-    };
-
-    void Send(const std::string& msg)
-    {
-        std::vector<pmix::info>* infos = new std::vector<pmix::info>();
-        infos->emplace_back("fairmq.cmd", msg);
-        PMIx_Notify_event(static_cast<int>(pmix::Command::general),
-                          &fProcess,
-                          PMIX_RANGE_NAMESPACE,
-                          infos->data(), infos->size(),
-                          &Commands::OpCompleteCallback<std::vector<pmix::info>>,
-                          infos);
-    }
-
-    void Send(const std::string& msg, rank rank)
-    {
-        pmix::proc destination(fProcess);
-        destination.rank = rank;
-        Send(msg, {destination});
-    }
-
-    void Send(const std::string& msg, const std::vector<proc>& destination)
-    {
-        std::unique_ptr<Holder> holder = std::make_unique<Holder>();
-
-        PMIX_DATA_ARRAY_CREATE(holder->fData, destination.size(), PMIX_PROC);
-        memcpy(holder->fData->array, destination.data(), destination.size() * sizeof(pmix_proc_t));
-        // LOG(warn) << "OLOG: " << msg << " > " << static_cast<pmix_proc_t*>(holder->fData->array)[0].nspace << ": " << static_cast<pmix_proc_t*>(holder->fData->array)[0].rank;
-        holder->fInfos.emplace_back(PMIX_EVENT_CUSTOM_RANGE, holder->fData);
-        // LOG(warn) << msg << " // packed range: " << static_cast<pmix_proc_t*>(static_cast<pmix_data_array_t*>(holder->fInfos.at(0).value.data.darray)->array)[0].nspace << "_" << static_cast<pmix_proc_t*>(static_cast<pmix_data_array_t*>(holder->fInfos.at(0).value.data.darray)->array)[0].rank;
-        // LOG(warn) << msg << " // packed range.type: " << pmix::typeNames.at(holder->fInfos.at(0).value.type);
-        // LOG(warn) << msg << " // packed range.array.type: " << pmix::typeNames.at(static_cast<pmix_data_array_t*>(holder->fInfos.at(0).value.data.darray)->type);
-        // LOG(warn) << msg << " // packed range.array.size: " << static_cast<pmix_data_array_t*>(holder->fInfos.at(0).value.data.darray)->size;
-        // LOG(warn) << holder->fInfos.size();
-        holder->fInfos.emplace_back("fairmq.cmd", msg);
-        // LOG(warn) << msg << " // packed msg: " << holder->fInfos.at(1).value.data.string;
-        // LOG(warn) << msg << " // packed msg.type: " << pmix::typeNames.at(holder->fInfos.at(1).value.type);
-        // LOG(warn) << holder->fInfos.size();
-
-        PMIx_Notify_event(static_cast<int>(pmix::Command::general),
-                          &fProcess,
-                          PMIX_RANGE_CUSTOM,
-                          holder->fInfos.data(), holder->fInfos.size(),
-                          &Commands::OpCompleteCallback<Holder>,
-                          holder.get());
-        holder.release();
-    }
-
-  private:
-    static void EventHandlerRegistration(pmix_status_t s, size_t handlerRef, void* obj)
-    {
-        if (s == PMIX_SUCCESS) {
-            LOG(debug) << "Successfully registered event handler, reference = " << static_cast<unsigned long>(handlerRef);
-            static_cast<Commands*>(obj)->fHandlerRef = handlerRef;
-            static_cast<Commands*>(obj)->fSubscribed = true;
-        } else {
-            LOG(error) << "Could not register PMIx event handler, status = " << s;
-        }
-        static_cast<Commands*>(obj)->fBlocker.Signal();
-    }
-
-    static void EventHandlerDeregistration(pmix_status_t s, void* obj)
-    {
-        if (s == PMIX_SUCCESS) {
-            LOG(debug) << "Successfully deregistered event handler, reference = " << static_cast<Commands*>(obj)->fHandlerRef;
-            static_cast<Commands*>(obj)->fSubscribed = false;
-        } else {
-            LOG(error) << "Could not deregister PMIx event handler, reference = " << static_cast<Commands*>(obj)->fHandlerRef << ", status = " << s;
-        }
-        static_cast<Commands*>(obj)->fBlocker.Signal();
-    }
-
-    template<typename T>
-    static void OpCompleteCallback(pmix_status_t s, void* data)
-    {
-        if (s == PMIX_SUCCESS) {
-            // LOG(info) << "Operation completed successfully";
-        } else {
-            LOG(error) << "Could not complete operation, status = " << s;
-        }
-        if (data) {
-            // LOG(warn) << "Destroying event data...";
-            delete static_cast<T*>(data);
-        }
-    }
-
-    static void Handler(size_t handlerId,
-                        pmix_status_t s,
-                        const pmix_proc_t* src,
-                        pmix_info_t info[], size_t ninfo,
-                        pmix_info_t[] /* results */, size_t nresults,
-                        pmix_event_notification_cbfunc_fn_t cbfunc,
-                        void* cbdata)
-    {
-        std::stringstream ss;
-        ss << "Event handler called with "
-           << "status: "    << s                               << ", "
-           << "source: "    << src->nspace << "_" << src->rank << ", "
-           << "ninfo: "     << ninfo                           << ", "
-           << "nresults: "  << nresults                        << ", "
-           << "handlerId: " << handlerId;
-
-        std::string msg;
-
-        Commands* obj = nullptr;
-
-        if (ninfo > 0) {
-            ss << ":\n";
-            for (size_t i = 0; i < ninfo; ++i) {
-                ss << "   [" << i << "]: key: '"        << info[i].key
-                                  << "', value: '"      << pmix::get_value_str(info[i].value)
-                                  << "', value.type: '" << pmix::typeNames.at(info[i].value.type)
-                                  << "', flags: "       << info[i].flags;
-
-                if (std::strcmp(info[i].key, "fairmq.cmd") == 0) {
-                    msg = pmix::get_value_str(info[i].value);
-                }
-
-                if (std::strcmp(info[i].key, PMIX_EVENT_RETURN_OBJECT) == 0) {
-                    obj = static_cast<Commands*>(info[i].value.data.ptr);
-                }
-
-                if (i < ninfo - 1) {
-                    ss << "\n";
-                }
-            }
-        }
-
-
-        if (obj != nullptr) {
-            if (static_cast<Commands*>(obj)->fProcess.rank != src->rank) {
-                // LOG(warn) << ss.str();
-                static_cast<Commands*>(obj)->fCallback(msg, proc(const_cast<char*>(src->nspace), rank(src->rank)));
-            } else {
-                // LOG(trace) << "suppressing message from itself";
-            }
-        } else {
-            LOG(error) << "ERROR";
-        }
-
-        if (cbfunc != nullptr) {
-            cbfunc(PMIX_SUCCESS, nullptr, 0, nullptr, nullptr, cbdata);
-        }
-    }
-
-    const proc& fProcess;
-    size_t fHandlerRef;
-    std::function<void(const std::string& msg, const proc& sender)> fCallback;
-    std::array<pmix_info_t, 1> fInfos;
-    bool fSubscribed;
-    fair::mq::tools::SharedSemaphore fBlocker;
-};
-
-} /* namespace pmix */
-
-#endif /* PMIXCOMMANDS_H */

fairmq/plugins/PMIx/PMIxPlugin.cxx

@@ -1,205 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2019-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include "PMIxPlugin.h"
-
-#include <fairmq/tools/Strings.h>
-
-#include <sstream>
-#include <stdexcept>
-#include <cstdint> // UINT32_MAX
-
-using namespace std;
-
-namespace fair::mq::plugins
-{
-
-PMIxPlugin::PMIxPlugin(const string& name,
-                       const Plugin::Version version,
-                       const string& maintainer,
-                       const string& homepage,
-                       PluginServices* pluginServices)
-    : Plugin(name, version, maintainer, homepage, pluginServices)
-    , fProcess(Init())
-    , fPid(getpid())
-    , fPMIxClient(tools::ToString("PMIx client(pid=", fPid, ") "))
-    , fDeviceId(string(fProcess.nspace) + "_" + to_string(fProcess.rank))
-    // , fLastExternalController(UINT32_MAX)
-    , fExitingAckedByLastExternalController(false)
-    , fCurrentState(DeviceState::Idle)
-    , fLastState(DeviceState::Idle)
-{
-    TakeDeviceControl();
-    LOG(debug) << PMIxClient() << "pmix::init() OK: " << fProcess << ", version=" << pmix::get_version();
-    SetProperty<string>("id", fDeviceId);
-
-    Fence("pmix::init");
-
-    // LOG(info) << "PMIX_EXTERNAL_ERR_BASE: " << PMIX_EXTERNAL_ERR_BASE;
-
-    // job level infos
-    // LOG(info) << "PMIX_SESSION_ID: "         << pmix::getInfo(PMIX_SESSION_ID, fProcess);
-    // LOG(info) << "PMIX_UNIV_SIZE: "          << pmix::getInfo(PMIX_UNIV_SIZE, fProcess);
-    // LOG(info) << "PMIX_JOB_SIZE: "           << pmix::getInfo(PMIX_JOB_SIZE, fProcess);
-    // LOG(info) << "PMIX_JOB_NUM_APPS: "       << pmix::getInfo(PMIX_JOB_NUM_APPS, fProcess);
-    // LOG(info) << "PMIX_APP_SIZE: "           << pmix::getInfo(PMIX_APP_SIZE, fProcess);
-    // LOG(info) << "PMIX_MAX_PROCS: "          << pmix::getInfo(PMIX_MAX_PROCS, fProcess);
-    // LOG(info) << "PMIX_NUM_NODES: "          << pmix::getInfo(PMIX_NUM_NODES, fProcess);
-    // LOG(info) << "PMIX_CLUSTER_ID: "         << pmix::getInfo(PMIX_CLUSTER_ID, fProcess);
-    // LOG(info) << "PMIX_NSPACE: "             << pmix::getInfo(PMIX_NSPACE, fProcess);
-    // LOG(info) << "PMIX_JOBID: "              << pmix::getInfo(PMIX_JOBID, fProcess);
-    // LOG(info) << "PMIX_NODE_LIST: "          << pmix::getInfo(PMIX_NODE_LIST, fProcess);
-    // LOG(info) << "PMIX_ALLOCATED_NODELIST: " << pmix::getInfo(PMIX_ALLOCATED_NODELIST, fProcess);
-    // LOG(info) << "PMIX_NPROC_OFFSET: "       << pmix::getInfo(PMIX_NPROC_OFFSET, fProcess);
-    // LOG(info) << "PMIX_LOCALLDR: "           << pmix::getInfo(PMIX_LOCALLDR, fProcess);
-    // LOG(info) << "PMIX_APPLDR: "             << pmix::getInfo(PMIX_APPLDR, fProcess);
-
-    // // per-node information
-    // LOG(info) << "PMIX_NODE_SIZE: "          << pmix::getInfo(PMIX_NODE_SIZE, fProcess);
-    // LOG(info) << "PMIX_LOCAL_SIZE: "         << pmix::getInfo(PMIX_LOCAL_SIZE, fProcess);
-    // LOG(info) << "PMIX_AVAIL_PHYS_MEMORY: "  << pmix::getInfo(PMIX_AVAIL_PHYS_MEMORY, fProcess);
-
-    // // per-process information
-    // LOG(info) << "PMIX_PROCID: "             << pmix::getInfo(PMIX_PROCID, fProcess);
-    // LOG(info) << "PMIX_APPNUM: "             << pmix::getInfo(PMIX_APPNUM, fProcess);
-    // LOG(info) << "PMIX_LOCAL_RANK: "         << pmix::getInfo(PMIX_LOCAL_RANK, fProcess);
-    // LOG(info) << "PMIX_NODE_RANK: "          << pmix::getInfo(PMIX_NODE_RANK, fProcess);
-    // LOG(info) << "PMIX_RANK: "               << pmix::getInfo(PMIX_RANK, fProcess);
-    // LOG(info) << "PMIX_GLOBAL_RANK: "        << pmix::getInfo(PMIX_GLOBAL_RANK, fProcess);
-    // LOG(info) << "PMIX_APP_RANK: "           << pmix::getInfo(PMIX_APP_RANK, fProcess);
-
-    SubscribeToDeviceStateChange([this](DeviceState newState) {
-        switch (newState) {
-            case DeviceState::Bound:
-                Publish();
-                break;
-            case DeviceState::Connecting:
-                Lookup();
-                break;
-            case DeviceState::Exiting:
-                ReleaseDeviceControl();
-                UnsubscribeFromDeviceStateChange();
-                break;
-            default:
-                break;
-        }
-
-        lock_guard<mutex> lock{fStateChangeSubscriberMutex};
-        fLastState = fCurrentState;
-        fCurrentState = newState;
-        // for (auto subscriberId : fStateChangeSubscribers) {
-            // LOG(debug) << "Publishing state-change: " << fLastState << "->" << newState << " to " << subscriberId;
-        // }
-    });
-}
-
-PMIxPlugin::~PMIxPlugin()
-{
-    LOG(debug) << "Destroying PMIxPlugin";
-    ReleaseDeviceControl();
-    while (pmix::initialized()) {
-        try {
-            pmix::finalize();
-            LOG(debug) << PMIxClient() << "pmix::finalize() OK";
-        } catch (const pmix::runtime_error& e) {
-            LOG(debug) << PMIxClient() << "pmix::finalize() failed: " << e.what();
-        }
-    }
-}
-
-auto PMIxPlugin::Init() -> pmix::proc
-{
-    if (!pmix::initialized()) {
-        return pmix::init();
-    } else {
-        throw runtime_error("trying to initialize PMIx while it is already initialized");
-    }
-}
-
-auto PMIxPlugin::Publish() -> void
-{
-    auto channels(GetChannelInfo());
-    vector<pmix::info> info;
-
-    for (const auto& c : channels) {
-        string methodKey("chans." + c.first + "." + to_string(c.second - 1) + ".method");
-        if (GetProperty<string>(methodKey) == "bind") {
-            for (int i = 0; i < c.second; ++i) {
-                string addressKey("chans." + c.first + "." + to_string(i) + ".address");
-                info.emplace_back(addressKey, GetProperty<string>(addressKey));
-                LOG(debug) << PMIxClient() << info.back();
-            }
-        }
-    }
-
-    if (info.size() > 0) {
-        pmix::publish(info);
-        LOG(debug) << PMIxClient() << "pmix::publish() OK: published " << info.size()
-                   << " binding channels.";
-    }
-}
-
-auto PMIxPlugin::Fence() -> void
-{
-    pmix::proc all(fProcess);
-    all.rank = pmix::rank::wildcard;
-
-    pmix::fence({all});
-}
-
-auto PMIxPlugin::Fence(const std::string& label) -> void
-{
-    Fence();
-    LOG(debug) << PMIxClient() << "pmix::fence() [" << label << "] OK";
-}
-
-auto PMIxPlugin::Lookup() -> void
-{
-    auto channels(GetChannelInfo());
-    for (const auto& c : channels) {
-        string methodKey("chans." + c.first + "." + to_string(c.second - 1) + ".method");
-        if (GetProperty<string>(methodKey) == "connect") {
-            for (int i = 0; i < c.second; ++i) {
-                vector<pmix::pdata> pdata;
-                string addressKey("chans." + c.first + "." + to_string(i) + ".address");
-                pdata.emplace_back();
-                pdata.back().set_key(addressKey);
-                vector<pmix::info> info;
-                info.emplace_back(PMIX_WAIT, static_cast<int>(pdata.size()));
-
-                if (pdata.size() > 0) {
-                    pmix::lookup(pdata, info);
-                    LOG(debug) << PMIxClient() << "pmix::lookup() OK";
-                }
-
-                for (const auto& p : pdata) {
-                    if (p.value.type == PMIX_UNDEF) {
-                        LOG(debug) << PMIxClient() << "pmix::lookup() not found: key=" << p.key;
-                    } else if (p.value.type == PMIX_STRING) {
-                        LOG(debug) << PMIxClient() << "pmix::lookup() found:"
-                                   << " key=" << p.key << ",value=" << p.value.data.string;
-                        SetProperty<string>(p.key, p.value.data.string);
-                    } else {
-                        LOG(debug) << PMIxClient() << "pmix::lookup() wrong type returned: "
-                                   << "key=" << p.key << ",type=" << p.value.type;
-                    }
-                }
-            }
-        }
-    }
-}
-
-auto PMIxPlugin::WaitForExitingAck() -> void
-{
-    unique_lock<mutex> lock(fStateChangeSubscriberMutex);
-    fExitingAcked.wait_for(lock, chrono::milliseconds(1000), [this]() {
-        return fExitingAckedByLastExternalController;
-    });
-}
-
-} // namespace fair::mq::plugins

fairmq/plugins/PMIx/PMIxPlugin.h

@@ -1,87 +0,0 @@
-/********************************************************************************
- * Copyright (C) 2019-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#ifndef FAIR_MQ_PLUGINS_PMIX
-#define FAIR_MQ_PLUGINS_PMIX
-
-#include "PMIx.hpp"
-#include "PMIxCommands.h"
-
-#include <fairmq/Plugin.h>
-#include <fairmq/Version.h>
-#include <fairlogger/Logger.h>
-
-#include <string>
-#include <sstream>
-#include <stdexcept>
-#include <string>
-#include <sys/types.h>
-#include <unistd.h>
-#include <vector>
-
-namespace fair::mq::plugins
-{
-
-class PMIxPlugin : public Plugin
-{
-  public:
-    PMIxPlugin(const std::string& name,
-         const Plugin::Version version,
-         const std::string& maintainer,
-         const std::string& homepage,
-         PluginServices* pluginServices);
-    ~PMIxPlugin();
-
-    auto PMIxClient() const -> std::string { return fPMIxClient; };
-
-  private:
-    pmix::proc fProcess;
-    pid_t fPid;
-    std::string fPMIxClient;
-    std::string fDeviceId;
-
-    std::set<uint32_t> fStateChangeSubscribers;
-    // uint32_t fLastExternalController;
-    bool fExitingAckedByLastExternalController;
-    std::condition_variable fExitingAcked;
-    std::mutex fStateChangeSubscriberMutex;
-
-    DeviceState fCurrentState;
-    DeviceState fLastState;
-
-    auto Init() -> pmix::proc;
-    auto Publish() -> void;
-    auto Fence() -> void;
-    auto Fence(const std::string& label) -> void;
-    auto Lookup() -> void;
-
-    auto WaitForExitingAck() -> void;
-};
-
-Plugin::ProgOptions PMIxProgramOptions()
-{
-    boost::program_options::options_description options("PMIx Plugin");
-    options.add_options()
-        ("pmix-dummy", boost::program_options::value<int>()->default_value(0), "Dummy.");
-    return options;
-}
-
-REGISTER_FAIRMQ_PLUGIN(
-    PMIxPlugin,                                  // Class name
-    pmix,                                        // Plugin name (string, lower case chars only)
-    (Plugin::Version{FAIRMQ_VERSION_MAJOR,
-                     FAIRMQ_VERSION_MINOR,
-                     FAIRMQ_VERSION_PATCH}),     // Version
-    "FairRootGroup <fairroot@gsi.de>",           // Maintainer
-    "https://github.com/FairRootGroup/FairMQ",   // Homepage
-    PMIxProgramOptions                           // custom program options for the plugin
-)
-
-} // namespace fair::mq::plugins
-
-#endif /* FAIR_MQ_PLUGINS_PMIX */

fairmq/plugins/PMIx/runPMIxCommandUI.cxx

@@ -1,131 +0,0 @@
-/********************************************************************************
- *  Copyright (C) 2014-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH *
- *                                                                              *
- *              This software is distributed under the terms of the             *
- *              GNU Lesser General Public Licence (LGPL) version 3,             *
- *                  copied verbatim in the file "LICENSE"                       *
- ********************************************************************************/
-
-#include <fairmq/States.h>
-
-#include <fairlogger/Logger.h>
-
-#include "PMIx.hpp"
-#include "PMIxCommands.h"
-
-#include <boost/program_options.hpp>
-
-#include <condition_variable>
-#include <algorithm>
-#include <cstdlib>
-#include <iostream>
-#include <mutex>
-#include <string>
-#include <termios.h> // raw mode console input
-#include <thread>
-#include <utility>
-#include <unistd.h>
-#include <vector>
-
-using namespace std;
-namespace bpo = boost::program_options;
-
-const std::map<fair::mq::Transition, fair::mq::State> expected =
-{
-    { fair::mq::Transition::InitDevice,   fair::mq::State::InitializingDevice },
-    { fair::mq::Transition::CompleteInit, fair::mq::State::Initialized },
-    { fair::mq::Transition::Bind,         fair::mq::State::Bound },
-    { fair::mq::Transition::Connect,      fair::mq::State::DeviceReady },
-    { fair::mq::Transition::InitTask,     fair::mq::State::Ready },
-    { fair::mq::Transition::Run,          fair::mq::State::Running },
-    { fair::mq::Transition::Stop,         fair::mq::State::Ready },
-    { fair::mq::Transition::ResetTask,    fair::mq::State::DeviceReady },
-    { fair::mq::Transition::ResetDevice,  fair::mq::State::Idle },
-    { fair::mq::Transition::End,          fair::mq::State::Exiting }
-};
-
-struct MiniTopo
-{
-    explicit MiniTopo(unsigned int n)
-        : fState(n, fair::mq::State::Ok)
-    {}
-
-    void WaitFor(const fair::mq::State state)
-    {
-        std::unique_lock<std::mutex> lk(fMtx);
-
-        fCV.wait(lk, [&](){
-            unsigned int count = std::count_if(fState.cbegin(), fState.cend(), [=](const auto& s) {
-                return s == state;
-            });
-
-            bool result = count == fState.size();
-            cout << "expecting " << state << " for " << fState.size() << " devices, found " << count << ", condition: " << result << endl;
-            return result;
-        });
-    }
-
-    void Update(uint32_t rank, const fair::mq::State state)
-    {
-        try {
-            {
-                std::lock_guard<std::mutex> lk(fMtx);
-                fState.at(rank - 1) = state;
-            }
-            fCV.notify_one();
-        } catch (const std::exception& e) {
-            LOG(error) << "Exception in Update: " << e.what();
-        }
-    }
-
-  private:
-    vector<fair::mq::State> fState;
-    std::mutex fMtx;
-    std::condition_variable fCV;
-};
-
-int main(int argc, char* argv[])
-{
-    try {
-        unsigned int numDevices;
-
-        bpo::options_description options("Common options");
-        options.add_options()
-            ("number-devices,n", bpo::value<unsigned int>(&numDevices)->default_value(0), "Number of devices (will be removed in the future)")
-            ("help,h", "Produce help message");
-
-        bpo::variables_map vm;
-        bpo::store(bpo::command_line_parser(argc, argv).options(options).run(), vm);
-
-        if (vm.count("help")) {
-            cout << "FairMQ DDS Command UI" << endl << options << endl;
-            cout << "Commands: [c] check state, [o] dump config, [h] help, [r] run, [s] stop, [t] reset task, [d] reset device, [q] end, [j] init task, [i] init device, [k] complete init, [b] bind, [x] connect" << endl;
-            return EXIT_SUCCESS;
-        }
-
-        bpo::notify(vm);
-
-        fair::Logger::SetConsoleSeverity(fair::Severity::debug);
-        fair::Logger::SetConsoleColor(true);
-        fair::Logger::SetVerbosity(fair::Verbosity::low);
-
-        pmix::proc process;
-
-        if (!pmix::initialized()) {
-            process = pmix::init();
-            LOG(warn) << "pmix::init() OK: " << process << ", version=" << pmix::get_version();
-        }
-
-        pmix::proc all(process);
-        all.rank = pmix::rank::wildcard;
-        pmix::fence({all});
-        LOG(warn) << "pmix::fence() [pmix::init] OK";
-
-        MiniTopo topo(numDevices);
-    } catch (exception& e) {
-        LOG(error) << "Error: " << e.what();
-        return EXIT_FAILURE;
-    }
-    LOG(warn) << "exiting";
-    return EXIT_SUCCESS;
-}

fairmq/plugins/config/Config.cxx

@@ -1,5 +1,5 @@
 /********************************************************************************
- * Copyright (C) 2017-2022 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
+ * Copyright (C) 2017-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -11,6 +11,7 @@
 #include <fairmq/JSONParser.h>
 #include <fairmq/SuboptParser.h>
 
+#include <cstddef>  // for std::size_t
 #include <vector>
 
 using namespace std;
@@ -40,7 +41,7 @@ Config::Config(const string& name, Plugin::Version version, const string& mainta
                         LOG(debug) << "channel-config: Parsing channel configuration";
                         SetProperties(SuboptParser(GetProperty<vector<string>>("channel-config"), idForParser));
                     } else {
-                        LOG(warn) << "fair::mq::plugins::Config: no channels configuration provided via --mq-config or --channel-config";
+                        LOG(info) << "fair::mq::plugins::Config: no channels configuration provided via --mq-config or --channel-config";
                     }
                 } catch (exception& e) {
                     LOG(error) << e.what();
@@ -72,11 +73,11 @@ Plugin::ProgOptions ConfigPluginProgramOptions()
         ("shm-zero-segment",              po::value<bool          >()->default_value(false),             "Shared memory: zero the shared memory segment memory after initialization (opened or created).")
         ("shm-zero-segment-on-creation",  po::value<bool          >()->default_value(false),             "Shared memory: zero the shared memory segment memory only once when created.")
         ("shm-throw-bad-alloc",           po::value<bool          >()->default_value(true),              "Shared memory: throw fair::mq::MessageBadAlloc if cannot allocate a message (retry if false).")
+        ("shm-metadata-msg-size",         po::value<std::size_t   >()->default_value(0),                 "Shared memory: size of the zmq metadata message (values smaller than minimum are clamped to the minimum).")
         ("bad-alloc-max-attempts",        po::value<int           >(),                                   "Maximum number of allocation attempts before throwing fair::mq::MessageBadAlloc. -1 is infinite. There is always at least one attempt, so 0 has safe effect as 1.")
         ("bad-alloc-attempt-interval",    po::value<int           >()->default_value(50),                "Interval between attempts if cannot allocate a message (in ms).")
-        ("shm-monitor",                   po::value<bool          >()->default_value(true),              "Shared memory: run monitor daemon.")
+        ("shm-monitor",                   po::value<bool          >()->default_value(false),             "Shared memory: run monitor daemon.")
         ("shm-no-cleanup",                po::value<bool          >()->default_value(false),             "Shared memory: do not cleanup the memory when last device leaves.")
-        ("ofi-size-hint",                 po::value<size_t        >()->default_value(0),                 "EXPERIMENTAL: OFI size hint for the allocator.")
         ("rate",                          po::value<float         >()->default_value(0.),                "Rate for conditional run loop (Hz).")
         ("session",                       po::value<string        >()->default_value("default"),         "Session name.")
         ("config-key",                    po::value<string        >(),                                   "Use provided value instead of device id for fetching the configuration from JSON file.")

fairmq/plugins/control/Control.cxx

@@ -489,12 +489,6 @@ auto Control::RunShutdownSequence() -> void
             case DeviceState::Running:
                 ChangeDeviceState(DeviceStateTransition::Stop);
                 break;
-            case DeviceState::Binding:
-            case DeviceState::Connecting:
-            case DeviceState::InitializingTask:
-            case DeviceState::ResettingTask:
-            case DeviceState::ResettingDevice:
-                ChangeDeviceState(DeviceStateTransition::Auto);
             default:
                 // LOG(debug) << "Controller ignoring event: " << nextState;
                 break;

fairmq/shmem/Common.h

@@ -13,6 +13,7 @@
 #include <functional> // std::equal_to
 
 #include <boost/functional/hash.hpp>
+#include <boost/interprocess/allocators/adaptive_pool.hpp>
 #include <boost/interprocess/allocators/allocator.hpp>
 #include <boost/interprocess/containers/map.hpp>
 #include <boost/interprocess/containers/string.hpp>
@@ -21,10 +22,12 @@
 #include <boost/interprocess/managed_shared_memory.hpp>
 #include <boost/interprocess/mem_algo/simple_seq_fit.hpp>
 #include <boost/unordered_map.hpp>
-#include <boost/variant.hpp>
+#include <variant>
 
 #include <sys/types.h>
 
+#include <fairmq/tools/Strings.h>
+
 namespace fair::mq::shmem
 {
 
@@ -41,6 +44,35 @@ using RBTreeBestFitSegment = boost::interprocess::basic_managed_shared_memory<ch
     boost::interprocess::null_index>;
     // boost::interprocess::iset_index>;
 
+inline std::string MakeShmName(const std::string& shmId, const std::string& type) {
+    return std::string("fmq_" + shmId + "_" + type);
+}
+
+inline std::string MakeShmName(const std::string& shmId, const std::string& type, int index) {
+    return std::string(MakeShmName(shmId, type) + "_" + std::to_string(index));
+}
+
+struct RefCount
+{
+    explicit RefCount(uint16_t c)
+        : count(c)
+    {}
+
+    uint16_t Get() { return count.load(); }
+    uint16_t Increment() { return count.fetch_add(1); }
+    uint16_t Decrement() { return count.fetch_sub(1); }
+
+    std::atomic<uint16_t> count;
+};
+
+// Number of nodes allocated at once when the allocator runs out of nodes.
+static constexpr size_t numNodesPerBlock = 4096;
+// Maximum number of totally free blocks that the adaptive node pool will hold.
+// The rest of the totally free blocks will be deallocated with the segment manager.
+static constexpr size_t maxFreeBlocks = 2;
+
+using RefCountPool = boost::interprocess::adaptive_pool<RefCount, boost::interprocess::managed_shared_memory::segment_manager, numNodesPerBlock, maxFreeBlocks>;
+
 using SegmentManager = boost::interprocess::managed_shared_memory::segment_manager;
 using VoidAlloc      = boost::interprocess::allocator<void, SegmentManager>;
 using CharAlloc      = boost::interprocess::allocator<char, SegmentManager>;
@@ -48,6 +80,90 @@ using Str            = boost::interprocess::basic_string<char, std::char_traits<
 using StrAlloc       = boost::interprocess::allocator<Str, SegmentManager>;
 using StrVector      = boost::interprocess::vector<Str, StrAlloc>;
 
+// ShmHeader stores user buffer alignment and the reference count in the following structure:
+// [HdrOffset(uint16_t)][Hdr alignment][Hdr][user buffer alignment][user buffer]
+// The alignment of Hdr depends on the alignment of std::atomic and is stored in the first entry
+struct ShmHeader
+{
+    struct Hdr
+    {
+        uint16_t userOffset;
+        std::atomic<uint16_t> refCount;
+    };
+
+    static Hdr* HdrPtr(char* ptr)
+    {
+        // [HdrOffset(uint16_t)][Hdr alignment][Hdr][user buffer alignment][user buffer]
+        //                                     ^
+        return reinterpret_cast<Hdr*>(ptr + sizeof(uint16_t) + *(reinterpret_cast<uint16_t*>(ptr)));
+    }
+
+    static uint16_t HdrPartSize() // [HdrOffset(uint16_t)][Hdr alignment][Hdr]
+    {
+        // [HdrOffset(uint16_t)][Hdr alignment][Hdr][user buffer alignment][user buffer]
+        // <--------------------------------------->
+        return sizeof(uint16_t) + alignof(Hdr) + sizeof(Hdr);
+    }
+
+    static std::atomic<uint16_t>& RefCountPtr(char* ptr)
+    {
+        // get the ref count ptr from the Hdr
+        return HdrPtr(ptr)->refCount;
+    }
+
+    static uint16_t UserOffset(char* ptr)
+    {
+        return HdrPartSize() + HdrPtr(ptr)->userOffset;
+    }
+
+    static char* UserPtr(char* ptr)
+    {
+        // [HdrOffset(uint16_t)][Hdr alignment][Hdr][user buffer alignment][user buffer]
+        //                                                                 ^
+        return ptr + HdrPartSize() + HdrPtr(ptr)->userOffset;
+    }
+
+    static uint16_t RefCount(char* ptr) { return RefCountPtr(ptr).load(); }
+    static uint16_t IncrementRefCount(char* ptr) { return RefCountPtr(ptr).fetch_add(1); }
+    static uint16_t DecrementRefCount(char* ptr) { return RefCountPtr(ptr).fetch_sub(1); }
+
+    static size_t FullSize(size_t size, size_t alignment)
+    {
+        // [HdrOffset(uint16_t)][Hdr alignment][Hdr][user buffer alignment][user buffer]
+        // <--------------------------------------------------------------------------->
+        return HdrPartSize() + alignment + size;
+    }
+
+    static void Construct(char* ptr, size_t alignment)
+    {
+        // place the Hdr in the aligned location, fill it and store its offset to HdrOffset
+
+        // the address alignment should be at least 2
+        assert(reinterpret_cast<uintptr_t>(ptr) % 2 == 0);
+
+        // offset to the beginning of the Hdr. store it in the beginning
+        uint16_t hdrOffset = alignof(Hdr) - ((reinterpret_cast<uintptr_t>(ptr) + sizeof(uint16_t)) % alignof(Hdr));
+        memcpy(ptr, &hdrOffset, sizeof(hdrOffset));
+
+        // offset to the beginning of the user buffer, store in Hdr together with the ref count
+        uint16_t userOffset = alignment - ((reinterpret_cast<uintptr_t>(ptr) + HdrPartSize()) % alignment);
+        new(ptr + sizeof(uint16_t) + hdrOffset) Hdr{ userOffset, std::atomic<uint16_t>(1) };
+    }
+
+    static void Destruct(char* ptr) { RefCountPtr(ptr).~atomic(); }
+};
+
+struct MetaHeader
+{
+    size_t fSize; // size of the shm buffer
+    size_t fHint; // user-defined value, given by the user on message creation and returned to the user on "buffer no longer needed"-callbacks
+    boost::interprocess::managed_shared_memory::handle_t fHandle; // handle to shm buffer, convertible to shm buffer ptr
+    mutable boost::interprocess::managed_shared_memory::handle_t fShared; // handle to the buffer storing the ref count for shared buffers
+    uint16_t fRegionId; // id of the unmanaged region
+    mutable uint16_t fSegmentId; // id of the managed segment
+    bool fManaged; // true = managed segment, false = unmanaged region
+};
+
 enum class AllocationAlgorithm : int
 {
     rbtree_best_fit,
@@ -56,26 +172,30 @@ enum class AllocationAlgorithm : int
 
 struct RegionInfo
 {
-    RegionInfo(const VoidAlloc& alloc)
-        : fPath("", alloc)
-        , fCreationFlags(0)
-        , fUserFlags(0)
-        , fSize(0)
-        , fDestroyed(false)
-    {}
+    static constexpr uint64_t DefaultRcSegmentSize = 10000000;
 
-    RegionInfo(const char* path, int flags, uint64_t userFlags, uint64_t size, const VoidAlloc& alloc)
+    RegionInfo(const char* path, int flags, uint64_t userFlags, uint64_t size, uint64_t rcSegmentSize, const VoidAlloc& alloc)
         : fPath(path, alloc)
         , fCreationFlags(flags)
         , fUserFlags(userFlags)
         , fSize(size)
+        , fRCSegmentSize(rcSegmentSize)
         , fDestroyed(false)
     {}
 
+    RegionInfo(const VoidAlloc& alloc)
+        : RegionInfo("", 0, 0, 0, DefaultRcSegmentSize, alloc)
+    {}
+
+    RegionInfo(const char* path, int flags, uint64_t userFlags, uint64_t size, const VoidAlloc& alloc)
+        : RegionInfo(path, flags, userFlags, size, DefaultRcSegmentSize, alloc)
+    {}
+
     Str fPath;
     int fCreationFlags;
     uint64_t fUserFlags;
     uint64_t fSize;
+    uint64_t fRCSegmentSize;
     bool fDestroyed;
 };
 
@@ -143,17 +263,6 @@ struct RegionCounter
     std::atomic<uint16_t> fCount;
 };
 
-struct MetaHeader
-{
-    size_t fSize;
-    size_t fHint;
-    boost::interprocess::managed_shared_memory::handle_t fHandle;
-    mutable boost::interprocess::managed_shared_memory::handle_t fShared;
-    uint16_t fRegionId;
-    mutable uint16_t fSegmentId;
-    bool fManaged;
-};
-
 #ifdef FAIRMQ_DEBUG_MODE
 struct MsgCounter
 {
@@ -219,73 +328,7 @@ std::string makeShmIdStr(const std::string& sessionId);
 std::string makeShmIdStr(uint64_t val);
 uint64_t makeShmIdUint64(const std::string& sessionId);
 
-
-struct SegmentSize : public boost::static_visitor<size_t>
-{
-    template<typename S>
-    size_t operator()(S& s) const { return s.get_size(); }
-};
-
-struct SegmentAddress : public boost::static_visitor<void*>
-{
-    template<typename S>
-    void* operator()(S& s) const { return s.get_address(); }
-};
-
-struct SegmentMemoryZeroer : public boost::static_visitor<>
-{
-    template<typename S>
-    void operator()(S& s) const { s.zero_free_memory(); }
-};
-
-struct SegmentFreeMemory : public boost::static_visitor<size_t>
-{
-    template<typename S>
-    size_t operator()(S& s) const { return s.get_free_memory(); }
-};
-
-struct SegmentHandleFromAddress : public boost::static_visitor<boost::interprocess::managed_shared_memory::handle_t>
-{
-    SegmentHandleFromAddress(const void* _ptr) : ptr(_ptr) {}
-
-    template<typename S>
-    boost::interprocess::managed_shared_memory::handle_t operator()(S& s) const { return s.get_handle_from_address(ptr); }
-
-    const void* ptr;
-};
-
-struct SegmentAddressFromHandle : public boost::static_visitor<char*>
-{
-    SegmentAddressFromHandle(const boost::interprocess::managed_shared_memory::handle_t _handle) : handle(_handle) {}
-
-    template<typename S>
-    char* operator()(S& s) const { return reinterpret_cast<char*>(s.get_address_from_handle(handle)); }
-
-    const boost::interprocess::managed_shared_memory::handle_t handle;
-};
-
-struct SegmentAllocate : public boost::static_visitor<char*>
-{
-    SegmentAllocate(const size_t _size) : size(_size) {}
-
-    template<typename S>
-    char* operator()(S& s) const { return reinterpret_cast<char*>(s.allocate(size)); }
-
-    const size_t size;
-};
-
-struct SegmentAllocateAligned : public boost::static_visitor<void*>
-{
-    SegmentAllocateAligned(const size_t _size, const size_t _alignment) : size(_size), alignment(_alignment) {}
-
-    template<typename S>
-    void* operator()(S& s) const { return s.allocate_aligned(size, alignment); }
-
-    const size_t size;
-    const size_t alignment;
-};
-
-struct SegmentBufferShrink : public boost::static_visitor<char*>
+struct SegmentBufferShrink
 {
     SegmentBufferShrink(const size_t _new_size, char* _local_ptr)
         : new_size(_new_size)
@@ -303,16 +346,6 @@ struct SegmentBufferShrink : public boost::static_visitor<char*>
     mutable char* local_ptr;
 };
 
-struct SegmentDeallocate : public boost::static_visitor<>
-{
-    SegmentDeallocate(char* _ptr) : ptr(_ptr) {}
-
-    template<typename S>
-    void operator()(S& s) const { return s.deallocate(ptr); }
-
-    char* ptr;
-};
-
 } // namespace fair::mq::shmem
 
 #endif /* FAIR_MQ_SHMEM_COMMON_H_ */

fairmq/shmem/Manager.h

@@ -1,5 +1,5 @@
 /********************************************************************************
- * Copyright (C) 2014-2021 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
+ * Copyright (C) 2014-2023 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH  *
  *                                                                              *
  *              This software is distributed under the terms of the             *
  *              GNU Lesser General Public Licence (LGPL) version 3,             *
@@ -19,18 +19,16 @@
 
 #include <fairlogger/Logger.h>
 
-#include <boost/date_time/posix_time/posix_time.hpp>
-#include <boost/filesystem.hpp>
 #include <boost/interprocess/ipc/message_queue.hpp>
 #include <boost/interprocess/managed_shared_memory.hpp>
 #include <boost/interprocess/sync/interprocess_condition.hpp>
 #include <boost/interprocess/sync/interprocess_mutex.hpp>
 #include <boost/interprocess/sync/named_mutex.hpp>
-#include <boost/variant.hpp>
 
 #include <algorithm> // max
+#include <chrono>
 #include <condition_variable>
-#include <cstddef> // max_align_t
+#include <cstddef> // max_align_t, std::size_t
 #include <cstdlib> // getenv
 #include <cstring> // memcpy
 #include <memory> // make_unique
@@ -43,6 +41,7 @@
 #include <tuple>
 #include <unordered_map>
 #include <utility> // pair
+#include <variant>
 #include <vector>
 
 #include <unistd.h> // getuid
@@ -52,79 +51,6 @@
 namespace fair::mq::shmem
 {
 
-// ShmHeader stores user buffer alignment and the reference count in the following structure:
-// [HdrOffset(uint16_t)][Hdr alignment][Hdr][user buffer alignment][user buffer]
-// The alignment of Hdr depends on the alignment of std::atomic and is stored in the first entry
-struct ShmHeader
-{
-    struct Hdr
-    {
-        uint16_t userOffset;
-        std::atomic<uint16_t> refCount;
-    };
-
-    static Hdr* HdrPtr(char* ptr)
-    {
-        // [HdrOffset(uint16_t)][Hdr alignment][Hdr][user buffer alignment][user buffer]
-        //                                     ^
-        return reinterpret_cast<Hdr*>(ptr + sizeof(uint16_t) + *(reinterpret_cast<uint16_t*>(ptr)));
-    }
-
-    static uint16_t HdrPartSize() // [HdrOffset(uint16_t)][Hdr alignment][Hdr]
-    {
-        // [HdrOffset(uint16_t)][Hdr alignment][Hdr][user buffer alignment][user buffer]
-        // <--------------------------------------->
-        return sizeof(uint16_t) + alignof(Hdr) + sizeof(Hdr);
-    }
-
-    static std::atomic<uint16_t>& RefCountPtr(char* ptr)
-    {
-        // get the ref count ptr from the Hdr
-        return HdrPtr(ptr)->refCount;
-    }
-
-    static uint16_t UserOffset(char* ptr)
-    {
-        return HdrPartSize() + HdrPtr(ptr)->userOffset;
-    }
-
-    static char* UserPtr(char* ptr)
-    {
-        // [HdrOffset(uint16_t)][Hdr alignment][Hdr][user buffer alignment][user buffer]
-        //                                                                 ^
-        return ptr + HdrPartSize() + HdrPtr(ptr)->userOffset;
-    }
-
-    static uint16_t RefCount(char* ptr) { return RefCountPtr(ptr).load(); }
-    static uint16_t IncrementRefCount(char* ptr) { return RefCountPtr(ptr).fetch_add(1); }
-    static uint16_t DecrementRefCount(char* ptr) { return RefCountPtr(ptr).fetch_sub(1); }
-
-    static size_t FullSize(size_t size, size_t alignment)
-    {
-        // [HdrOffset(uint16_t)][Hdr alignment][Hdr][user buffer alignment][user buffer]
-        // <--------------------------------------------------------------------------->
-        return HdrPartSize() + alignment + size;
-    }
-
-    static void Construct(char* ptr, size_t alignment)
-    {
-        // place the Hdr in the aligned location, fill it and store its offset to HdrOffset
-
-        // the address alignment should be at least 2
-        assert(reinterpret_cast<uintptr_t>(ptr) % 2 == 0);
-
-        // offset to the beginning of the Hdr. store it in the beginning
-        uint16_t hdrOffset = alignof(Hdr) - ((reinterpret_cast<uintptr_t>(ptr) + sizeof(uint16_t)) % alignof(Hdr));
-        memcpy(ptr, &hdrOffset, sizeof(hdrOffset));
-
-        // offset to the beginning of the user buffer, store in Hdr together with the ref count
-        uint16_t userOffset = alignment - ((reinterpret_cast<uintptr_t>(ptr) + HdrPartSize()) % alignment);
-        new(ptr + sizeof(uint16_t) + hdrOffset) Hdr{ userOffset, std::atomic<uint16_t>(1) };
-    }
-
-    static void Destruct(char* ptr) { RefCountPtr(ptr).~atomic(); }
-};
-
 class Manager
 {
   public:
@@ -132,7 +58,7 @@ class Manager
         : fShmId64(config ? config->GetProperty<uint64_t>("shmid", makeShmIdUint64(sessionName)) : makeShmIdUint64(sessionName))
         , fShmId(makeShmIdStr(fShmId64))
         , fSegmentId(config ? config->GetProperty<uint16_t>("shm-segment-id", 0) : 0)
-        , fManagementSegment(boost::interprocess::open_or_create, std::string("fmq_" + fShmId + "_mng").c_str(), kManagementSegmentSize)
+        , fManagementSegment(boost::interprocess::open_or_create, MakeShmName(fShmId, "mng").c_str(), kManagementSegmentSize)
         , fShmVoidAlloc(fManagementSegment.get_segment_manager())
         , fShmMtx(fManagementSegment.find_or_construct<boost::interprocess::interprocess_mutex>(boost::interprocess::unique_instance)())
         , fNumObservedEvents(0)
@@ -152,6 +78,7 @@ class Manager
         , fBadAllocMaxAttempts(1)
         , fBadAllocAttemptIntervalInMs(config ? config->GetProperty<int>("bad-alloc-attempt-interval", 50) : 50)
         , fNoCleanup(config ? config->GetProperty<bool>("shm-no-cleanup", false) : false)
+        , fMetadataMsgSize(config ? config->GetProperty<std::size_t>("shm-metadata-msg-size", 0) : 0)
     {
         using namespace boost::interprocess;
 
@@ -207,22 +134,22 @@ class Manager
 
             fEventCounter = fManagementSegment.find<EventCounter>(unique_instance).first;
             if (fEventCounter) {
-                LOG(debug) << "event counter found: " << fEventCounter->fCount;
+                LOG(trace) << "event counter found: " << fEventCounter->fCount;
             } else {
-                LOG(debug) << "no event counter found, creating one and initializing with 0";
+                LOG(trace) << "no event counter found, creating one and initializing with 0";
                 fEventCounter = fManagementSegment.construct<EventCounter>(unique_instance)(0);
-                LOG(debug) << "initialized event counter with: " << fEventCounter->fCount;
+                LOG(trace) << "initialized event counter with: " << fEventCounter->fCount;
             }
 
             fDeviceCounter = fManagementSegment.find<DeviceCounter>(unique_instance).first;
             if (fDeviceCounter) {
-                LOG(debug) << "device counter found, with value of " << fDeviceCounter->fCount << ". incrementing.";
+                LOG(trace) << "device counter found, with value of " << fDeviceCounter->fCount << ". incrementing.";
                 (fDeviceCounter->fCount)++;
-                LOG(debug) << "incremented device counter, now: " << fDeviceCounter->fCount;
+                LOG(trace) << "incremented device counter, now: " << fDeviceCounter->fCount;
             } else {
-                LOG(debug) << "no device counter found, creating one and initializing with 1";
+                LOG(trace) << "no device counter found, creating one and initializing with 1";
                 fDeviceCounter = fManagementSegment.construct<DeviceCounter>(unique_instance)(1);
-                LOG(debug) << "initialized device counter with: " << fDeviceCounter->fCount;
+                LOG(trace) << "initialized device counter with: " << fDeviceCounter->fCount;
             }
 
             fShmSegments = fManagementSegment.find_or_construct<Uint16SegmentInfoHashMap>(unique_instance)(fShmVoidAlloc);
@@ -231,7 +158,7 @@ class Manager
             bool createdSegment = false;
 
             try {
-                std::string segmentName("fmq_" + fShmId + "_m_" + std::to_string(fSegmentId));
+                std::string segmentName = MakeShmName(fShmId, "m", fSegmentId);
                 auto it = fShmSegments->find(fSegmentId);
                 if (it == fShmSegments->end()) {
                     // no segment with given id exists, creating
@@ -265,10 +192,10 @@ class Manager
                         }
                     }
                 }
-                LOG(debug) << "Created/opened shared memory segment '" << "fmq_" << fShmId << "_m_" << fSegmentId << "'."
-                << " Size: " << boost::apply_visitor(SegmentSize(), fSegments.at(fSegmentId)) << " bytes."
-                << " Available: " << boost::apply_visitor(SegmentFreeMemory(), fSegments.at(fSegmentId)) << " bytes."
-                << " Allocation algorithm: " << allocationAlgorithm;
+                LOG(debug) << (createdSegment ? "Created" : "Opened") << " managed shared memory segment " << "fmq_" << fShmId << "_m_" << fSegmentId
+                    << ". Size: " << std::visit([](auto& s) { return s.get_size(); }, fSegments.at(fSegmentId)) << " bytes."
+                    << " Available: " << std::visit([](auto& s) { return s.get_free_memory(); }, fSegments.at(fSegmentId)) << " bytes."
+                    << " Allocation algorithm: " << allocationAlgorithm;
             } catch (interprocess_exception& bie) {
                 LOG(error) << "Failed to create/open shared memory segment '" << "fmq_" << fShmId << "_m_" << fSegmentId << "': " << bie.what();
                 throw TransportError(tools::ToString("Failed to create/open shared memory segment '", "fmq_", fShmId, "_m_", fSegmentId, "': ", bie.what()));
@@ -305,14 +232,16 @@ class Manager
     void ZeroSegment(uint16_t id)
     {
         LOG(debug) << "Zeroing the managed segment free memory...";
-        boost::apply_visitor(SegmentMemoryZeroer(), fSegments.at(id));
+        std::visit([](auto& s) { return s.zero_free_memory(); }, fSegments.at(id));
         LOG(debug) << "Successfully zeroed the managed segment free memory.";
     }
 
     void MlockSegment(uint16_t id)
     {
         LOG(debug) << "Locking the managed segment memory pages...";
-        if (mlock(boost::apply_visitor(SegmentAddress(), fSegments.at(id)), boost::apply_visitor(SegmentSize(), fSegments.at(id))) == -1) {
+        if (mlock(
+                std::visit([](auto& s) { return s.get_address(); }, fSegments.at(id)),
+                std::visit([](auto& s) { return s.get_size(); }, fSegments.at(id))) == -1) {
             LOG(error) << "Could not lock the managed segment memory. Code: " << errno << ", reason: " << strerror(errno);
             throw TransportError(tools::ToString("Could not lock the managed segment memory: ", strerror(errno)));
         }
@@ -327,7 +256,7 @@ class Manager
     {
         using namespace boost::interprocess;
         try {
-            named_mutex monitorStatus(open_only, std::string("fmq_" + id + "_ms").c_str());
+            named_mutex monitorStatus(open_only, MakeShmName(id, "ms").c_str());
             LOG(debug) << "Found fairmq-shmmonitor for shared memory id " << id;
         } catch (interprocess_exception&) {
             LOG(debug) << "no fairmq-shmmonitor found for shared memory id " << id << ", starting...";
@@ -336,7 +265,7 @@ class Manager
                 int numTries = 0;
                 do {
                     try {
-                        named_mutex monitorStatus(open_only, std::string("fmq_" + id + "_ms").c_str());
+                        named_mutex monitorStatus(open_only, MakeShmName(id, "ms").c_str());
                         LOG(debug) << "Started fairmq-shmmonitor for shared memory id " << id;
                         break;
                     } catch (interprocess_exception&) {
@@ -395,20 +324,32 @@ class Manager
 
                 const uint16_t id = cfg.id.value();
 
+                std::lock_guard<std::mutex> lock(fLocalRegionsMtx);
+
                 UnmanagedRegion* region = nullptr;
-                bool newRegionCreated = false;
-                {
-                    std::lock_guard<std::mutex> lock(fLocalRegionsMtx);
-                    auto res = fRegions.emplace(id, std::make_unique<UnmanagedRegion>(fShmId, size, false, cfg));
-                    newRegionCreated = res.second;
+
+                auto it = fRegions.find(id);
+                if (it != fRegions.end()) {
+                    region = it->second.get();
+                    if (region->fControlling) {
+                        LOG(error) << "Unmanaged Region with id " << id << " already exists. Only unique IDs per session are allowed.";
+                        throw TransportError(tools::ToString("Unmanaged Region with id ", id, " already exists. Only unique IDs per session are allowed."));
+                    }
+
+                    LOG(debug) << "Unmanaged region (view) already present, promoting to controller";
+                    region->BecomeController(cfg);
+                } else {
+                    // we need to update local config, if the region information already exists
+                    auto info = fShmRegions->find(id);
+                    if (info != fShmRegions->end()) {
+                        cfg.rcSegmentSize = info->second.fRCSegmentSize;
+                    }
+
+                    auto res = fRegions.emplace(id, std::make_unique<UnmanagedRegion>(fShmId, size, true, cfg));
                     region = res.first->second.get();
                 }
                 // LOG(debug) << "Created region with id '" << id << "', path: '" << cfg.path << "', flags: '" << cfg.creationFlags << "'";
 
-                if (!newRegionCreated) {
-                    region->fRemote = false; // TODO: this should be more clear, refactor it.
-                }
-
                 // start ack receiver only if a callback has been provided.
                 if (callback || bulkCallback) {
                     region->SetCallbacks(callback, bulkCallback);
@@ -429,7 +370,7 @@ class Manager
         }
     }
 
-    UnmanagedRegion* GetRegion(uint16_t id)
+    UnmanagedRegion* GetRegionFromCache(uint16_t id)
     {
         // NOTE: gcc optimizations. Prevent loading tls addresses many times in the fast path
         const auto &lTlCache = fTlRegionCache;
@@ -443,60 +384,59 @@ class Manager
             }
         }
 
-        boost::interprocess::scoped_lock<boost::interprocess::interprocess_mutex> shmLock(*fShmMtx);
         // slow path: check invalidation
         if (lTlCacheGen != fRegionsGen) {
             fTlRegionCache.fRegionsTLCache.clear();
         }
 
-        std::lock_guard<std::mutex> lock(fLocalRegionsMtx);
-        auto* lRegion = GetRegionUnsafe(id, shmLock);
+        auto* lRegion = GetRegion(id);
         fTlRegionCache.fRegionsTLCache.emplace_back(std::make_tuple(lRegion, id, fShmId64));
         fTlRegionCache.fRegionsTLCacheGen = fRegionsGen;
         return lRegion;
     }
 
-    UnmanagedRegion* GetRegionUnsafe(uint16_t id, boost::interprocess::scoped_lock<boost::interprocess::interprocess_mutex>& lockedShmLock)
+    UnmanagedRegion* GetRegion(uint16_t id)
     {
-        // remote region could actually be a local one if a message originates from this device (has been sent out and returned)
+        std::lock_guard<std::mutex> lock(fLocalRegionsMtx);
         auto it = fRegions.find(id);
         if (it != fRegions.end()) {
             return it->second.get();
-        } else {
-            try {
-                // get region info
+        }
+
+        try {
+            RegionConfig cfg;
+            // get region info
+            {
+                boost::interprocess::scoped_lock<boost::interprocess::interprocess_mutex> shmLock(*fShmMtx);
                 RegionInfo regionInfo = fShmRegions->at(id);
-                // safe to unlock now - no shm container accessed after this
-                lockedShmLock.unlock();
-                RegionConfig cfg;
                 cfg.id = id;
                 cfg.creationFlags = regionInfo.fCreationFlags;
+                cfg.rcSegmentSize = regionInfo.fRCSegmentSize;
                 cfg.path = regionInfo.fPath.c_str();
-                // LOG(debug) << "Located remote region with id '" << id << "', path: '" << cfg.path << "', flags: '" << cfg.creationFlags << "'";
-
-                auto r = fRegions.emplace(id, std::make_unique<UnmanagedRegion>(fShmId, 0, true, std::move(cfg)));
-                r.first->second->InitializeQueues();
-                r.first->second->StartAckSender();
-                lockedShmLock.lock();
-                return r.first->second.get();
-            } catch (std::out_of_range& oor) {
-                LOG(error) << "Could not get remote region with id '" << id << "'. Does the region creator run with the same session id?";
-                LOG(error) << oor.what();
-                return nullptr;
-            } catch (boost::interprocess::interprocess_exception& e) {
-                LOG(error) << "Could not get remote region for id '" << id << "': " << e.what();
-                return nullptr;
             }
+            // LOG(debug) << "Located remote region with id '" << id << "', path: '" << cfg.path << "', flags: '" << cfg.creationFlags << "'";
+
+            auto r = fRegions.emplace(id, std::make_unique<UnmanagedRegion>(fShmId, 0, false, std::move(cfg)));
+            r.first->second->InitializeQueues();
+            r.first->second->StartAckSender();
+            return r.first->second.get();
+        } catch (std::out_of_range& oor) {
+            LOG(error) << "Could not get remote region with id '" << id << "'. Does the region creator run with the same session id?";
+            LOG(error) << oor.what();
+            return nullptr;
+        } catch (boost::interprocess::interprocess_exception& e) {
+            LOG(error) << "Could not get remote region for id '" << id << "': " << e.what();
+            return nullptr;
         }
     }
 
     void RemoveRegion(uint16_t id)
     {
         try {
+            boost::interprocess::scoped_lock<boost::interprocess::interprocess_mutex> shmLock(*fShmMtx);
             std::lock_guard<std::mutex> lock(fLocalRegionsMtx);
             fRegions.at(id)->StopAcks();
             {
-                boost::interprocess::scoped_lock<boost::interprocess::interprocess_mutex> shmLock(*fShmMtx);
                 if (fRegions.at(id)->RemoveOnDestruction()) {
                     fShmRegions->at(id).fDestroyed = true;
                     (fEventCounter->fCount)++;
@@ -512,44 +452,74 @@ class Manager
     std::vector<fair::mq::RegionInfo> GetRegionInfo()
     {
         std::vector<fair::mq::RegionInfo> result;
-        boost::interprocess::scoped_lock<boost::interprocess::interprocess_mutex> shmLock(*fShmMtx);
+        std::map<uint64_t, RegionConfig> regionCfgs;
 
-        for (const auto& e : *fShmSegments) {
-            // make sure any segments in the session are found
-            GetSegment(e.first);
-            try {
+        {
+            boost::interprocess::scoped_lock<boost::interprocess::interprocess_mutex> shmLock(*fShmMtx);
+
+            for (const auto& [segmentId, segmentInfo] : *fShmSegments) {
+                // make sure any segments in the session are found
+                GetSegment(segmentId);
+                try {
+                    fair::mq::RegionInfo info;
+                    info.managed = true;
+                    info.id = segmentId;
+                    info.event = RegionEvent::created;
+                    info.ptr = std::visit([](auto& s) { return s.get_address(); }, fSegments.at(segmentId));
+                    info.size = std::visit([](auto& s) { return s.get_size(); }, fSegments.at(segmentId));
+                    result.push_back(info);
+                } catch (const std::out_of_range& oor) {
+                    LOG(error) << "could not find segment with id " << segmentId;
+                    LOG(error) << oor.what();
+                }
+            }
+
+            for (const auto& [regionId, regionInfo] : *fShmRegions) {
                 fair::mq::RegionInfo info;
-                info.managed = true;
-                info.id = e.first;
-                info.event = RegionEvent::created;
-                info.ptr = boost::apply_visitor(SegmentAddress(), fSegments.at(e.first));
-                info.size = boost::apply_visitor(SegmentSize(), fSegments.at(e.first));
+                info.managed = false;
+                info.id = regionId;
+                info.flags = regionInfo.fUserFlags;
+                info.event = regionInfo.fDestroyed ? RegionEvent::destroyed : RegionEvent::created;
+                if (info.event == RegionEvent::created) {
+                    RegionConfig cfg;
+                    cfg.id = info.id;
+                    cfg.creationFlags = regionInfo.fCreationFlags;
+                    cfg.path = regionInfo.fPath.c_str();
+                    cfg.rcSegmentSize = regionInfo.fRCSegmentSize;
+                    regionCfgs.emplace(info.id, cfg);
+                    // fill the ptr+size info after shmLock is released, to avoid constructing local region under it
+                } else {
+                    info.ptr = nullptr;
+                    info.size = 0;
+                }
                 result.push_back(info);
-            } catch (const std::out_of_range& oor) {
-                LOG(error) << "could not find segment with id " << e.first;
-                LOG(error) << oor.what();
             }
         }
 
-        for (const auto& e : *fShmRegions) {
-            fair::mq::RegionInfo info;
-            info.managed = false;
-            info.id = e.first;
-            info.flags = e.second.fUserFlags;
-            info.event = e.second.fDestroyed ? RegionEvent::destroyed : RegionEvent::created;
-            if (info.event == RegionEvent::created) {
-                auto region = GetRegionUnsafe(info.id, shmLock);
-                if (region) {
+        // do another iteration outside of shm lock, to fill ptr+size of unmanaged regions
+        for (auto& info : result) {
+            if (!info.managed && info.event == RegionEvent::created) {
+                auto cfgIt = regionCfgs.find(info.id);
+                if (cfgIt != regionCfgs.end()) {
+                    UnmanagedRegion* region = nullptr;
+                    std::lock_guard<std::mutex> lock(fLocalRegionsMtx);
+                    auto it = fRegions.find(info.id);
+                    if (it != fRegions.end()) {
+                        region = it->second.get();
+                    } else {
+                        auto r = fRegions.emplace(cfgIt->first, std::make_unique<UnmanagedRegion>(fShmId, 0, false, cfgIt->second));
+                        region = r.first->second.get();
+                        region->InitializeQueues();
+                        region->StartAckSender();
+                    }
+
                     info.ptr = region->GetData();
                     info.size = region->GetSize();
                 } else {
-                    throw std::runtime_error(tools::ToString("GetRegionInfo() could not get region with id '", info.id, "'"));
+                    info.ptr = nullptr;
+                    info.size = 0;
                 }
-            } else {
-                info.ptr = nullptr;
-                info.size = 0;
             }
-            result.push_back(info);
         }
 
         return result;
@@ -677,9 +647,9 @@ class Manager
                 using namespace boost::interprocess;
 
                 if (segmentInfo.fAllocationAlgorithm == AllocationAlgorithm::rbtree_best_fit) {
-                    fSegments.emplace(id, RBTreeBestFitSegment(open_only, std::string("fmq_" + fShmId + "_m_" + std::to_string(id)).c_str()));
+                    fSegments.emplace(id, RBTreeBestFitSegment(open_only, MakeShmName(fShmId, "m", id).c_str()));
                 } else {
-                    fSegments.emplace(id, SimpleSeqFitSegment(open_only, std::string("fmq_" + fShmId + "_m_" + std::to_string(id)).c_str()));
+                    fSegments.emplace(id, SimpleSeqFitSegment(open_only, MakeShmName(fShmId, "m", id).c_str()));
                 }
             } catch (std::out_of_range& oor) {
                 LOG(error) << "Could not get segment with id '" << id << "': " << oor.what();
@@ -691,11 +661,11 @@ class Manager
 
     boost::interprocess::managed_shared_memory::handle_t GetHandleFromAddress(const void* ptr, uint16_t segmentId) const
     {
-        return boost::apply_visitor(SegmentHandleFromAddress(ptr), fSegments.at(segmentId));
+        return std::visit([ptr](auto& s) { return s.get_handle_from_address(ptr); }, fSegments.at(segmentId));
     }
     char* GetAddressFromHandle(const boost::interprocess::managed_shared_memory::handle_t handle, uint16_t segmentId) const
     {
-        return boost::apply_visitor(SegmentAddressFromHandle(handle), fSegments.at(segmentId));
+        return std::visit([handle](auto& s) { return reinterpret_cast<char*>(s.get_address_from_handle(handle)); }, fSegments.at(segmentId));
     }
 
     char* Allocate(size_t size, size_t alignment = 0)
@@ -708,21 +678,32 @@ class Manager
 
         while (!ptr) {
             try {
-                size_t segmentSize = boost::apply_visitor(SegmentSize(), fSegments.at(fSegmentId));
+                size_t segmentSize = std::visit([](auto& s) { return s.get_size(); }, fSegments.at(fSegmentId));
                 if (fullSize > segmentSize) {
                     throw MessageBadAlloc(tools::ToString("Requested message size (", fullSize, ") exceeds segment size (", segmentSize, ")"));
                 }
 
-                ptr = boost::apply_visitor(SegmentAllocate{fullSize}, fSegments.at(fSegmentId));
+                ptr = std::visit([fullSize](auto& s) { return reinterpret_cast<char*>(s.allocate(fullSize)); }, fSegments.at(fSegmentId));
                 ShmHeader::Construct(ptr, alignment);
             } catch (boost::interprocess::bad_alloc& ba) {
                 // LOG(warn) << "Shared memory full...";
                 if (fBadAllocMaxAttempts >= 0 && ++numAttempts >= fBadAllocMaxAttempts) {
-                    throw MessageBadAlloc(tools::ToString("shmem: could not create a message of size ", size, ", alignment: ", (alignment != 0) ? std::to_string(alignment) : "default", ", free memory: ", boost::apply_visitor(SegmentFreeMemory(), fSegments.at(fSegmentId))));
+                    throw MessageBadAlloc(tools::ToString("shmem: could not create a message of size ", size,
+                        ", alignment: ", (alignment != 0) ? std::to_string(alignment) : "default",
+                        ", free memory: ", std::visit([](auto& s) { return s.get_free_memory(); }, fSegments.at(fSegmentId))));
+                }
+                if (numAttempts == 1 && fBadAllocMaxAttempts > 1) {
+                    LOG(warn) << tools::ToString("shmem: could not create a message of size ", size,
+                        ", alignment: ", (alignment != 0) ? std::to_string(alignment) : "default",
+                        ", free memory: ", std::visit([](auto& s) { return s.get_free_memory(); }, fSegments.at(fSegmentId)),
+                        ". Will try ", (fBadAllocMaxAttempts > 1 ? (std::to_string(fBadAllocMaxAttempts - 1)) + " more times" : " until success"),
+                        ", in ", fBadAllocAttemptIntervalInMs, "ms intervals");
                 }
                 std::this_thread::sleep_for(std::chrono::milliseconds(fBadAllocAttemptIntervalInMs));
                 if (Interrupted()) {
-                    throw MessageBadAlloc(tools::ToString("shmem: could not create a message of size ", size, ", alignment: ", (alignment != 0) ? std::to_string(alignment) : "default", ", free memory: ", boost::apply_visitor(SegmentFreeMemory(), fSegments.at(fSegmentId))));
+                    throw MessageBadAlloc(tools::ToString("shmem: could not create a message of size ", size,
+                        ", alignment: ", (alignment != 0) ? std::to_string(alignment) : "default",
+                        ", free memory: ", std::visit([](auto& s) { return s.get_free_memory(); }, fSegments.at(fSegmentId))));
                 } else {
                     continue;
                 }
@@ -756,12 +737,12 @@ class Manager
         }
 #endif
         ShmHeader::Destruct(ptr);
-        boost::apply_visitor(SegmentDeallocate(ptr), fSegments.at(segmentId));
+        std::visit([ptr](auto& s) { s.deallocate(ptr); }, fSegments.at(segmentId));
     }
 
     char* ShrinkInPlace(size_t newSize, char* localPtr, uint16_t segmentId)
     {
-        return boost::apply_visitor(SegmentBufferShrink(newSize, localPtr), fSegments.at(segmentId));
+        return std::visit(SegmentBufferShrink(newSize, localPtr), fSegments.at(segmentId));
     }
 
     uint16_t GetSegmentId() const { return fSegmentId; }
@@ -793,6 +774,8 @@ class Manager
         }
     }
 
+    auto GetMetadataMsgSize() const noexcept { return fMetadataMsgSize; }
+
     ~Manager()
     {
         fRegionsGen += 1; // signal TL cache invalidation
@@ -807,7 +790,7 @@ class Manager
     uint64_t fShmId64;
     std::string fShmId;
     uint16_t fSegmentId;
-    std::unordered_map<uint16_t, boost::variant<RBTreeBestFitSegment, SimpleSeqFitSegment>> fSegments; // TODO: refactor to use Segment class
+    std::unordered_map<uint16_t, std::variant<RBTreeBestFitSegment, SimpleSeqFitSegment>> fSegments; // TODO: refactor to use Segment class
     boost::interprocess::managed_shared_memory fManagementSegment; // TODO: refactor to use ManagementSegment class
     VoidAlloc fShmVoidAlloc;
     boost::interprocess::interprocess_mutex* fShmMtx;
@@ -849,6 +832,8 @@ class Manager
     int fBadAllocMaxAttempts;
     int fBadAllocAttemptIntervalInMs;
     bool fNoCleanup;
+
+    std::size_t fMetadataMsgSize;
 };
 
 } // namespace fair::mq::shmem

fairmq/shmem/Message.h

@@ -36,60 +36,34 @@ class Message final : public fair::mq::Message
 
   public:
     Message(Manager& manager, fair::mq::TransportFactory* factory = nullptr)
-        : fair::mq::Message(factory)
-        , fManager(manager)
-        , fQueued(false)
-        , fMeta{0, 0, -1, -1, 0, fManager.GetSegmentId(), true}
-        , fRegionPtr(nullptr)
-        , fLocalPtr(nullptr)
-    {
-        fManager.IncrementMsgCounter();
-    }
+        : Message(manager, Alignment{0}, factory)
+    {}
 
-    Message(Manager& manager, Alignment alignment, fair::mq::TransportFactory* factory = nullptr)
+    Message(Manager& manager, Alignment /* alignment */, fair::mq::TransportFactory* factory = nullptr)
         : fair::mq::Message(factory)
         , fManager(manager)
-        , fQueued(false)
-        , fMeta{0, 0, -1, -1, 0, fManager.GetSegmentId(), true}
-        , fAlignment(alignment.alignment)
-        , fRegionPtr(nullptr)
-        , fLocalPtr(nullptr)
+        , fSegmentId(fManager.GetSegmentId())
     {
         fManager.IncrementMsgCounter();
     }
 
     Message(Manager& manager, const size_t size, fair::mq::TransportFactory* factory = nullptr)
-        : fair::mq::Message(factory)
-        , fManager(manager)
-        , fQueued(false)
-        , fMeta{0, 0, -1, -1, 0, fManager.GetSegmentId(), true}
-        , fRegionPtr(nullptr)
-        , fLocalPtr(nullptr)
-    {
-        InitializeChunk(size);
-        fManager.IncrementMsgCounter();
-    }
+        : Message(manager, size, Alignment{0}, factory)
+    {}
 
     Message(Manager& manager, const size_t size, Alignment alignment, fair::mq::TransportFactory* factory = nullptr)
         : fair::mq::Message(factory)
         , fManager(manager)
-        , fQueued(false)
-        , fMeta{0, 0, -1, -1, 0, fManager.GetSegmentId(), true}
-        , fAlignment(alignment.alignment)
-        , fRegionPtr(nullptr)
-        , fLocalPtr(nullptr)
+        , fSegmentId(fManager.GetSegmentId())
     {
-        InitializeChunk(size, fAlignment);
+        InitializeChunk(size, alignment.alignment);
         fManager.IncrementMsgCounter();
     }
 
     Message(Manager& manager, void* data, const size_t size, fair::mq::FreeFn* ffn, void* hint = nullptr, fair::mq::TransportFactory* factory = nullptr)
         : fair::mq::Message(factory)
         , fManager(manager)
-        , fQueued(false)
-        , fMeta{0, 0, -1, -1, 0, fManager.GetSegmentId(), true}
-        , fRegionPtr(nullptr)
-        , fLocalPtr(nullptr)
+        , fSegmentId(fManager.GetSegmentId())
     {
         if (InitializeChunk(size)) {
             std::memcpy(fLocalPtr, data, size);
@@ -105,10 +79,12 @@ class Message final : public fair::mq::Message
     Message(Manager& manager, UnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0, fair::mq::TransportFactory* factory = nullptr)
         : fair::mq::Message(factory)
         , fManager(manager)
-        , fQueued(false)
-        , fMeta{size, reinterpret_cast<size_t>(hint), -1, -1, static_cast<UnmanagedRegionImpl*>(region.get())->fRegionId, fManager.GetSegmentId(), false}
-        , fRegionPtr(nullptr)
         , fLocalPtr(static_cast<char*>(data))
+        , fSize(size)
+        , fHint(reinterpret_cast<size_t>(hint))
+        , fRegionId(static_cast<UnmanagedRegionImpl*>(region.get())->fRegionId)
+        , fSegmentId(fManager.GetSegmentId())
+        , fManaged(false)
     {
         if (region->GetType() != GetType()) {
             LOG(error) << "region type (" << region->GetType() << ") does not match message type (" << GetType() << ")";
@@ -117,7 +93,7 @@ class Message final : public fair::mq::Message
 
         if (reinterpret_cast<const char*>(data) >= reinterpret_cast<const char*>(region->GetData()) &&
             reinterpret_cast<const char*>(data) <= reinterpret_cast<const char*>(region->GetData()) + region->GetSize()) {
-            fMeta.fHandle = (boost::interprocess::managed_shared_memory::handle_t)(reinterpret_cast<const char*>(data) - reinterpret_cast<const char*>(region->GetData()));
+            fHandle = (boost::interprocess::managed_shared_memory::handle_t)(reinterpret_cast<const char*>(data) - reinterpret_cast<const char*>(region->GetData()));
         } else {
             LOG(error) << "trying to create region message with data from outside the region";
             throw TransportError("trying to create region message with data from outside the region");
@@ -128,10 +104,13 @@ class Message final : public fair::mq::Message
     Message(Manager& manager, MetaHeader& hdr, fair::mq::TransportFactory* factory = nullptr)
         : fair::mq::Message(factory)
         , fManager(manager)
-        , fQueued(false)
-        , fMeta{hdr}
-        , fRegionPtr(nullptr)
-        , fLocalPtr(nullptr)
+        , fSize(hdr.fSize)
+        , fHint(hdr.fHint)
+        , fHandle(hdr.fHandle)
+        , fShared(hdr.fShared)
+        , fRegionId(hdr.fRegionId)
+        , fSegmentId(hdr.fSegmentId)
+        , fManaged(hdr.fManaged)
     {
         fManager.IncrementMsgCounter();
     }
@@ -147,11 +126,10 @@ class Message final : public fair::mq::Message
         fQueued = false;
     }
 
-    void Rebuild(Alignment alignment) override
+    void Rebuild(Alignment /* alignment */) override
     {
         CloseMessage();
         fQueued = false;
-        fAlignment = alignment.alignment;
     }
 
     void Rebuild(size_t size) override
@@ -165,8 +143,7 @@ class Message final : public fair::mq::Message
     {
         CloseMessage();
         fQueued = false;
-        fAlignment = alignment.alignment;
-        InitializeChunk(size, fAlignment);
+        InitializeChunk(size, alignment.alignment);
     }
 
     void Rebuild(void* data, size_t size, fair::mq::FreeFn* ffn, void* hint = nullptr) override
@@ -187,17 +164,17 @@ class Message final : public fair::mq::Message
     void* GetData() const override
     {
         if (!fLocalPtr) {
-            if (fMeta.fManaged) {
-                if (fMeta.fSize > 0) {
-                    fManager.GetSegment(fMeta.fSegmentId);
-                    fLocalPtr = ShmHeader::UserPtr(fManager.GetAddressFromHandle(fMeta.fHandle, fMeta.fSegmentId));
+            if (fManaged) {
+                if (fSize > 0) {
+                    fManager.GetSegment(fSegmentId);
+                    fLocalPtr = ShmHeader::UserPtr(fManager.GetAddressFromHandle(fHandle, fSegmentId));
                 } else {
                     fLocalPtr = nullptr;
                 }
             } else {
-                fRegionPtr = fManager.GetRegion(fMeta.fRegionId);
+                fRegionPtr = fManager.GetRegionFromCache(fRegionId);
                 if (fRegionPtr) {
-                    fLocalPtr = reinterpret_cast<char*>(fRegionPtr->GetData()) + fMeta.fHandle;
+                    fLocalPtr = reinterpret_cast<char*>(fRegionPtr->GetData()) + fHandle;
                 } else {
                     // LOG(warn) << "could not get pointer from a region message";
                     fLocalPtr = nullptr;
@@ -208,37 +185,41 @@ class Message final : public fair::mq::Message
         return static_cast<void*>(fLocalPtr);
     }
 
-    size_t GetSize() const override { return fMeta.fSize; }
+    size_t GetSize() const override { return fSize; }
 
-    bool SetUsedSize(size_t newSize) override
+    bool SetUsedSize(size_t newSize, Alignment alignment = Alignment{0}) override
     {
-        if (newSize == fMeta.fSize) {
+        if (newSize == fSize) {
             return true;
         } else if (newSize == 0) {
             Deallocate();
             return true;
-        } else if (newSize <= fMeta.fSize) {
+        } else if (newSize <= fSize) {
             try {
+                char* oldPtr = fManager.GetAddressFromHandle(fHandle, fSegmentId);
                 try {
-                    char* oldPtr = fManager.GetAddressFromHandle(fMeta.fHandle, fMeta.fSegmentId);
                     uint16_t userOffset = ShmHeader::UserOffset(oldPtr);
-                    char* ptr = fManager.ShrinkInPlace(userOffset + newSize, oldPtr, fMeta.fSegmentId);
+                    char* ptr = fManager.ShrinkInPlace(userOffset + newSize, oldPtr, fSegmentId);
                     fLocalPtr = ShmHeader::UserPtr(ptr);
-                    fMeta.fSize = newSize;
+                    fSize = newSize;
                     return true;
                 } catch (boost::interprocess::bad_alloc& e) {
                     // if shrinking fails (can happen due to boost alignment requirements):
                     // unused size >= 1000000 bytes: reallocate fully
                     // unused size < 1000000 bytes: simply reset the size and keep the rest of the buffer until message destruction
-                    if (fMeta.fSize - newSize >= 1000000) {
-                        char* ptr = fManager.Allocate(newSize, fAlignment);
+                    if (fSize - newSize >= 1000000) {
+                        if (alignment.alignment == 0) {
+                            // if no alignment is provided, take the minimum alignment of the old pointer, but no more than 4096
+                            alignment.alignment = 1 << std::min(__builtin_ctz(reinterpret_cast<size_t>(oldPtr)), 12);
+                        }
+                        char* ptr = fManager.Allocate(newSize, alignment.alignment);
                         char* userPtr = ShmHeader::UserPtr(ptr);
                         std::memcpy(userPtr, fLocalPtr, newSize);
-                        fManager.Deallocate(fMeta.fHandle, fMeta.fSegmentId);
+                        fManager.Deallocate(fHandle, fSegmentId);
                         fLocalPtr = userPtr;
-                        fMeta.fHandle = fManager.GetHandleFromAddress(ptr, fMeta.fSegmentId);
+                        fHandle = fManager.GetHandleFromAddress(ptr, fSegmentId);
                     }
-                    fMeta.fSize = newSize;
+                    fSize = newSize;
                     return true;
                 }
             } catch (boost::interprocess::interprocess_exception& e) {
@@ -255,101 +236,130 @@ class Message final : public fair::mq::Message
 
     uint16_t GetRefCount() const
     {
-        if (fMeta.fHandle < 0) {
+        if (fHandle < 0) {
             return 1;
         }
 
-        if (fMeta.fManaged) { // managed segment
-            fManager.GetSegment(fMeta.fSegmentId);
-            return ShmHeader::RefCount(fManager.GetAddressFromHandle(fMeta.fHandle, fMeta.fSegmentId));
-        } else { // unmanaged region
-            if (fMeta.fShared < 0) { // UR msg is not yet shared
-                return 1;
-            } else {
-                fManager.GetSegment(fMeta.fSegmentId);
-                return ShmHeader::RefCount(fManager.GetAddressFromHandle(fMeta.fShared, fMeta.fSegmentId));
-            }
+        if (fManaged) { // managed segment
+            fManager.GetSegment(fSegmentId);
+            return ShmHeader::RefCount(fManager.GetAddressFromHandle(fHandle, fSegmentId));
         }
+        if (fShared < 0) { // UR msg is not yet shared
+            return 1;
+        }
+        fRegionPtr = fManager.GetRegionFromCache(fRegionId);
+        if (!fRegionPtr) {
+            throw TransportError(tools::ToString("Cannot get unmanaged region with id ", fRegionId));
+        }
+        return fRegionPtr->GetRefCountAddressFromHandle(fShared)->Get();
     }
 
     void Copy(const fair::mq::Message& other) override
     {
         const Message& otherMsg = static_cast<const Message&>(other);
-        if (otherMsg.fMeta.fHandle < 0) {
-            // if the other message is not initialized, close this one too and return
+        // if the other message is not initialized, close this one too and return
+        if (otherMsg.fHandle < 0) {
             CloseMessage();
             return;
         }
 
-        if (fMeta.fHandle >= 0) {
-            // if this msg is already initialized, close it first
+        // if this msg is already initialized, close it first
+        if (fHandle >= 0) {
             CloseMessage();
         }
 
-        if (otherMsg.fMeta.fManaged) { // managed segment
-            fMeta = otherMsg.fMeta;
-            fManager.GetSegment(fMeta.fSegmentId);
-            ShmHeader::IncrementRefCount(fManager.GetAddressFromHandle(fMeta.fHandle, fMeta.fSegmentId));
-        } else { // unmanaged region
-            if (otherMsg.fMeta.fShared < 0) { // if UR msg is not yet shared
-                // TODO: minimize the size to 0 and don't create extra space for user buffer alignment
-                char* ptr = fManager.Allocate(2, 0);
-                // point the fShared in the unmanaged region message to the refCount holder
-                otherMsg.fMeta.fShared = fManager.GetHandleFromAddress(ptr, fMeta.fSegmentId);
-                // the message needs to be able to locate in which segment the refCount is stored
-                otherMsg.fMeta.fSegmentId = fMeta.fSegmentId;
-                // point this message to the same content as the unmanaged region message
-                fMeta = otherMsg.fMeta;
-                // increment the refCount
-                ShmHeader::IncrementRefCount(ptr);
-            } else { // if the UR msg is already shared
-                fMeta = otherMsg.fMeta;
-                fManager.GetSegment(fMeta.fSegmentId);
-                ShmHeader::IncrementRefCount(fManager.GetAddressFromHandle(fMeta.fShared, fMeta.fSegmentId));
+        // increment ref count
+        if (otherMsg.fManaged) { // msg in managed segment
+            fManager.GetSegment(otherMsg.fSegmentId);
+            ShmHeader::IncrementRefCount(fManager.GetAddressFromHandle(otherMsg.fHandle, otherMsg.fSegmentId));
+        } else { // msg in unmanaged region
+            fRegionPtr = fManager.GetRegionFromCache(otherMsg.fRegionId);
+            if (!fRegionPtr) {
+                throw TransportError(tools::ToString("Cannot get unmanaged region with id ", otherMsg.fRegionId));
+            }
+            if (otherMsg.fShared < 0) {
+                // UR msg not yet shared, create the reference counting object with count 2
+                try {
+                    otherMsg.fShared = fRegionPtr->HandleFromAddress(&(fRegionPtr->MakeRefCount(2)));
+                } catch (boost::interprocess::bad_alloc& ba) {
+                    throw RefCountBadAlloc(tools::ToString(
+                        "Insufficient space in the reference count segment ",
+                        otherMsg.fRegionId,
+                        ", original exception: bad_alloc: ", 
+                        ba.what()));
+                }
+            } else {
+                fRegionPtr->GetRefCountAddressFromHandle(otherMsg.fShared)->Increment();
             }
         }
+
+        // copy meta data
+        fSize = otherMsg.fSize;
+        fHint = otherMsg.fHint;
+        fHandle = otherMsg.fHandle;
+        fShared = otherMsg.fShared;
+        fRegionId = otherMsg.fRegionId;
+        fSegmentId = otherMsg.fSegmentId;
+        fManaged = otherMsg.fManaged;
     }
 
     ~Message() override { CloseMessage(); }
 
   private:
     Manager& fManager;
-    bool fQueued;
-    MetaHeader fMeta;
-    size_t fAlignment;
-    mutable UnmanagedRegion* fRegionPtr;
-    mutable char* fLocalPtr;
+    mutable UnmanagedRegion* fRegionPtr = nullptr;
+    mutable char* fLocalPtr = nullptr;
+    size_t fSize = 0; // size of the shm buffer
+    size_t fHint = 0; // user-defined value, given by the user on message creation and returned to the user on "buffer no longer needed"-callbacks
+    boost::interprocess::managed_shared_memory::handle_t fHandle = -1; // handle to shm buffer, convertible to shm buffer ptr
+    mutable boost::interprocess::managed_shared_memory::handle_t fShared = -1; // handle to the buffer storing the ref count for shared buffers
+    uint16_t fRegionId = 0; // id of the unmanaged region
+    mutable uint16_t fSegmentId; // id of the managed segment
+    bool fManaged = true; // true = managed segment, false = unmanaged region
+    bool fQueued = false;
+
+    void SetMeta(const MetaHeader& meta)
+    {
+        fSize = meta.fSize;
+        fHint = meta.fHint;
+        fHandle = meta.fHandle;
+        fShared = meta.fShared;
+        fRegionId = meta.fRegionId;
+        fSegmentId = meta.fSegmentId;
+        fManaged = meta.fManaged;
+    }
 
     char* InitializeChunk(const size_t size, size_t alignment = 0)
     {
         if (size == 0) {
-            fMeta.fSize = 0;
+            fSize = 0;
             return fLocalPtr;
         }
         char* ptr = fManager.Allocate(size, alignment);
-        fMeta.fHandle = fManager.GetHandleFromAddress(ptr, fMeta.fSegmentId);
-        fMeta.fSize = size;
+        fHandle = fManager.GetHandleFromAddress(ptr, fSegmentId);
+        fSize = size;
         fLocalPtr = ShmHeader::UserPtr(ptr);
         return fLocalPtr;
     }
 
     void Deallocate()
     {
-        if (fMeta.fHandle >= 0 && !fQueued) {
-            if (fMeta.fManaged) { // managed segment
-                fManager.GetSegment(fMeta.fSegmentId);
-                uint16_t refCount = ShmHeader::DecrementRefCount(fManager.GetAddressFromHandle(fMeta.fHandle, fMeta.fSegmentId));
+        if (fHandle >= 0 && !fQueued) {
+            if (fManaged) { // managed segment
+                fManager.GetSegment(fSegmentId);
+                uint16_t refCount = ShmHeader::DecrementRefCount(fManager.GetAddressFromHandle(fHandle, fSegmentId));
                 if (refCount == 1) {
-                    fManager.Deallocate(fMeta.fHandle, fMeta.fSegmentId);
+                    fManager.Deallocate(fHandle, fSegmentId);
                 }
             } else { // unmanaged region
-                if (fMeta.fShared >= 0) {
-                    // make sure segment is initialized in this transport
-                    fManager.GetSegment(fMeta.fSegmentId);
-                    // release unmanaged region block if ref count is one
-                    uint16_t refCount = ShmHeader::DecrementRefCount(fManager.GetAddressFromHandle(fMeta.fShared, fMeta.fSegmentId));
+                if (fShared >= 0) {
+                    fRegionPtr = fManager.GetRegionFromCache(fRegionId);
+                    if (!fRegionPtr) {
+                        throw TransportError(tools::ToString("Cannot get unmanaged region with id ", fRegionId));
+                    }
+                    uint16_t refCount = fRegionPtr->GetRefCountAddressFromHandle(fShared)->Decrement();
                     if (refCount == 1) {
-                        fManager.Deallocate(fMeta.fShared, fMeta.fSegmentId);
+                        fRegionPtr->RemoveRefCount(*(fRegionPtr->GetRefCountAddressFromHandle(fShared)));
                         ReleaseUnmanagedRegionBlock();
                     }
                 } else {
@@ -357,21 +367,21 @@ class Message final : public fair::mq::Message
                 }
             }
         }
-        fMeta.fHandle = -1;
+        fHandle = -1;
         fLocalPtr = nullptr;
-        fMeta.fSize = 0;
+        fSize = 0;
     }
 
     void ReleaseUnmanagedRegionBlock()
     {
         if (!fRegionPtr) {
-            fRegionPtr = fManager.GetRegion(fMeta.fRegionId);
+            fRegionPtr = fManager.GetRegionFromCache(fRegionId);
         }
 
         if (fRegionPtr) {
-            fRegionPtr->ReleaseBlock({fMeta.fHandle, fMeta.fSize, fMeta.fHint});
+            fRegionPtr->ReleaseBlock({fHandle, fSize, fHint});
         } else {
-            LOG(warn) << "region ack queue for id " << fMeta.fRegionId << " no longer exist. Not sending ack";
+            LOG(warn) << "region ack queue for id " << fRegionId << " no longer exist. Not sending ack";
         }
     }
 
@@ -379,7 +389,6 @@ class Message final : public fair::mq::Message
     {
         try {
             Deallocate();
-            fAlignment = 0;
             fManager.DecrementMsgCounter();
         } catch (SharedMemoryError& sme) {
             LOG(error) << "error closing message: " << sme.what();

fairmq/shmem/Monitor.cxx

@@ -23,12 +23,14 @@
 #include <boost/interprocess/ipc/message_queue.hpp>
 
 #include <csignal>
+#include <cstdio>
 #include <iostream>
 #include <iomanip>
 #include <chrono>
 #include <ctime>
 #include <iomanip>
 #include <sstream>
+#include <variant>
 
 #include <poll.h>
 
@@ -73,7 +75,7 @@ Monitor::Monitor(string shmId, bool selfDestruct, bool interactive, bool viewOnl
 {
     if (!fViewOnly) {
         try {
-            bipc::named_mutex monitorStatus(bipc::create_only, string("fmq_" + fShmId + "_ms").c_str());
+            bipc::named_mutex monitorStatus(bipc::create_only, MakeShmName(fShmId, "ms").c_str());
         } catch (bie&) {
             if (fInteractive) {
                 LOG(error) << "fairmq-shmmonitor for shm id " << fShmId << " is already running. Try `fairmq-shmmonitor --cleanup --shmid " << fShmId << "`, or run in view-only mode (-v)";
@@ -131,7 +133,7 @@ void Monitor::Watch()
         using namespace boost::interprocess;
 
         try {
-            managed_shared_memory managementSegment(open_read_only, std::string("fmq_" + fShmId + "_mng").c_str());
+            managed_shared_memory managementSegment(open_read_only, MakeShmName(fShmId, "mng").c_str());
 
             fSeenOnce = true;
 
@@ -179,11 +181,11 @@ bool Monitor::PrintShm(const ShmId& shmId)
     using namespace boost::interprocess;
 
     try {
-        managed_shared_memory managementSegment(open_read_only, std::string("fmq_" + shmId.shmId + "_mng").c_str());
+        managed_shared_memory managementSegment(open_read_only, MakeShmName(shmId.shmId, "mng").c_str());
         VoidAlloc allocInstance(managementSegment.get_segment_manager());
 
         Uint16SegmentInfoHashMap* shmSegments = managementSegment.find<Uint16SegmentInfoHashMap>(unique_instance).first;
-        std::unordered_map<uint16_t, boost::variant<RBTreeBestFitSegment, SimpleSeqFitSegment>> segments;
+        std::unordered_map<uint16_t, std::variant<RBTreeBestFitSegment, SimpleSeqFitSegment>> segments;
 
         Uint16RegionInfoHashMap* shmRegions = managementSegment.find<Uint16RegionInfoHashMap>(unique_instance).first;
 
@@ -198,9 +200,9 @@ bool Monitor::PrintShm(const ShmId& shmId)
 
         for (const auto& s : *shmSegments) {
             if (s.second.fAllocationAlgorithm == AllocationAlgorithm::rbtree_best_fit) {
-                segments.emplace(s.first, RBTreeBestFitSegment(open_read_only, std::string("fmq_" + shmId.shmId + "_m_" + to_string(s.first)).c_str()));
+                segments.emplace(s.first, RBTreeBestFitSegment(open_read_only, MakeShmName(shmId.shmId, "m", s.first).c_str()));
             } else {
-                segments.emplace(s.first, SimpleSeqFitSegment(open_read_only, std::string("fmq_" + shmId.shmId + "_m_" + to_string(s.first)).c_str()));
+                segments.emplace(s.first, SimpleSeqFitSegment(open_read_only, MakeShmName(shmId.shmId, "m", s.first).c_str()));
             }
         }
 
@@ -233,8 +235,8 @@ bool Monitor::PrintShm(const ShmId& shmId)
            << ", managed segments:\n";
 
         for (const auto& s : segments) {
-            size_t free = boost::apply_visitor(SegmentFreeMemory(), s.second);
-            size_t total = boost::apply_visitor(SegmentSize(), s.second);
+            size_t free = std::visit([](auto& seg){ return seg.get_free_memory(); }, s.second);
+            size_t total = std::visit([](auto& seg){ return seg.get_size(); }, s.second);
             size_t used = total - free;
 
             std::string msgCount;
@@ -266,12 +268,19 @@ bool Monitor::PrintShm(const ShmId& shmId)
 
         if (shmRegions && !shmRegions->empty()) {
             ss << "\n   unmanaged regions:";
-            for (const auto& r : *shmRegions) {
-                ss << "\n      [" << r.first << "]: " << (r.second.fDestroyed ? "destroyed" : "alive");
-                ss << ", size: " << r.second.fSize;
+            for (const auto& [id, info] : *shmRegions) {
+                ss << "\n      [" << id << "]: " << (info.fDestroyed ? "destroyed" : "alive");
+                ss << ", size: " << info.fSize;
+
+                try {
+                    managed_shared_memory rcCountSegment(open_read_only, MakeShmName(shmId.shmId, "rrc", id).c_str());
+                    ss << ", rcCountSegment size: " << rcCountSegment.get_size() << ", free: " << rcCountSegment.get_free_memory();
+                } catch (bie&) {
+                    ss << ", rcCountSegment: not found";
+                }
 
                 // try {
-                //     boost::interprocess::message_queue q(open_only, std::string("fmq_" + std::string(shmId) + "_rgq_" + to_string(r.first)).c_str());
+                //     boost::interprocess::message_queue q(open_only, std::string("fmq_" + std::string(shmId) + "_rgq_" + to_string(id)).c_str());
                 //     ss << ", ack queue: " << q.get_num_msg() << " messages";
                 // } catch (bie&) {
                 //     ss << ", ack queue: not found";
@@ -324,7 +333,7 @@ void Monitor::CheckHeartbeats()
     while (!fTerminating) {
         std::this_thread::sleep_for(std::chrono::milliseconds(200));
         try {
-            managed_shared_memory managementSegment(open_read_only, std::string("fmq_" + fShmId + "_mng").c_str());
+            managed_shared_memory managementSegment(open_read_only, MakeShmName(fShmId, "mng").c_str());
             Heartbeat* hb = managementSegment.find<Heartbeat>(unique_instance).first;
 
             if (hb) {
@@ -406,7 +415,7 @@ void Monitor::Interactive()
 void Monitor::PrintDebugInfo(const ShmId& shmId __attribute__((unused)))
 {
 #ifdef FAIRMQ_DEBUG_MODE
-    string managementSegmentName("fmq_" + shmId.shmId + "_mng");
+    string managementSegmentName = MakeShmName(shmId.shmId, "mng");
     try {
         bipc::managed_shared_memory managementSegment(bipc::open_only, managementSegmentName.c_str());
         bipc::interprocess_mutex* mtx(managementSegment.find_or_construct<bipc::interprocess_mutex>(bipc::unique_instance)());
@@ -458,7 +467,7 @@ unordered_map<uint16_t, std::vector<BufferDebugInfo>> Monitor::GetDebugInfo(cons
     unordered_map<uint16_t, std::vector<BufferDebugInfo>> result;
 
 #ifdef FAIRMQ_DEBUG_MODE
-    string managementSegmentName("fmq_" + shmId.shmId + "_mng");
+    string managementSegmentName = MakeShmName(shmId.shmId, "mng");
     try {
         bipc::managed_shared_memory managementSegment(bipc::open_only, managementSegmentName.c_str());
         bipc::interprocess_mutex* mtx(managementSegment.find_or_construct<bipc::interprocess_mutex>(bipc::unique_instance)());
@@ -498,7 +507,7 @@ unsigned long Monitor::GetFreeMemory(const ShmId& shmId, uint16_t segmentId)
 {
     using namespace boost::interprocess;
     try {
-        bipc::managed_shared_memory managementSegment(bipc::open_only, std::string("fmq_" + shmId.shmId + "_mng").c_str());
+        bipc::managed_shared_memory managementSegment(bipc::open_only, MakeShmName(shmId.shmId, "mng").c_str());
         boost::interprocess::interprocess_mutex* mtx(managementSegment.find_or_construct<bipc::interprocess_mutex>(bipc::unique_instance)());
         boost::interprocess::scoped_lock<bipc::interprocess_mutex> lock(*mtx);
 
@@ -512,10 +521,10 @@ unsigned long Monitor::GetFreeMemory(const ShmId& shmId, uint16_t segmentId)
         auto it = shmSegments->find(segmentId);
         if (it != shmSegments->end()) {
             if (it->second.fAllocationAlgorithm == AllocationAlgorithm::rbtree_best_fit) {
-                RBTreeBestFitSegment segment(open_read_only, std::string("fmq_" + shmId.shmId + "_m_" + std::to_string(segmentId)).c_str());
+                RBTreeBestFitSegment segment(open_read_only, MakeShmName(shmId.shmId, "m", segmentId).c_str());
                 return segment.get_free_memory();
             } else {
-                SimpleSeqFitSegment segment(open_read_only, std::string("fmq_" + shmId.shmId + "_m_" + std::to_string(segmentId)).c_str());
+                SimpleSeqFitSegment segment(open_read_only, MakeShmName(shmId.shmId, "m", segmentId).c_str());
                 return segment.get_free_memory();
             }
         } else {
@@ -533,6 +542,88 @@ unsigned long Monitor::GetFreeMemory(const SessionId& sessionId, uint16_t segmen
     return GetFreeMemory(shmId, segmentId);
 }
 
+bool Monitor::SegmentIsPresent(const ShmId& shmId, uint16_t segmentId)
+{
+    using namespace boost::interprocess;
+    try {
+        bipc::managed_shared_memory managementSegment(bipc::open_read_only, MakeShmName(shmId.shmId, "mng").c_str());
+        Uint16SegmentInfoHashMap* shmSegments = managementSegment.find<Uint16SegmentInfoHashMap>(unique_instance).first;
+
+        if (!shmSegments) {
+            LOG(error) << "Found management segment, but could not locate segment info";
+            return false;
+        }
+
+        auto it = shmSegments->find(segmentId);
+        if (it != shmSegments->end()) {
+            try {
+                if (it->second.fAllocationAlgorithm == AllocationAlgorithm::rbtree_best_fit) {
+                    RBTreeBestFitSegment segment(open_read_only, MakeShmName(shmId.shmId, "m", segmentId).c_str());
+                } else {
+                    SimpleSeqFitSegment segment(open_read_only, MakeShmName(shmId.shmId, "m", segmentId).c_str());
+                }
+            } catch (bie&) {
+                LOG(error) << "Could not find segment with id '" << segmentId << "' for shmId '" << shmId.shmId << "'";
+                return false;
+            }
+        } else {
+            LOG(error) << "Could not find segment info for segment id '" << segmentId << "' for shmId '" << shmId.shmId << "'";
+            return false;
+        }
+    } catch (bie&) {
+        LOG(error) << "Could not find management segment for shmid '" << shmId.shmId << "'";
+        return false;
+    }
+
+    return true;
+}
+bool Monitor::SegmentIsPresent(const SessionId& sessionId, uint16_t segmentId)
+{
+    ShmId shmId{makeShmIdStr(sessionId.sessionId)};
+    return SegmentIsPresent(shmId, segmentId);
+}
+
+bool Monitor::RegionIsPresent(const ShmId& shmId, uint16_t regionId)
+{
+    using namespace boost::interprocess;
+    try {
+        bipc::managed_shared_memory managementSegment(bipc::open_read_only, MakeShmName(shmId.shmId, "mng").c_str());
+        Uint16RegionInfoHashMap* shmRegions = managementSegment.find<Uint16RegionInfoHashMap>(bipc::unique_instance).first;
+
+        if (!shmRegions) {
+            LOG(error) << "Found management segment, but could not locate region info";
+            return false;
+        }
+
+        std::string regionFileName(MakeShmName(shmId.shmId, "rg", regionId));
+
+        auto it = shmRegions->find(regionId);
+        if (it != shmRegions->end()) {
+            try {
+                if (it->second.fPath.empty()) {
+                    shared_memory_object object(open_only, regionFileName.c_str(), read_only);
+                }
+            } catch (bie&) {
+                LOG(error) << "Could not find region with id '" << regionId << "' for shmId '" << shmId.shmId << "'";
+                return false;
+            }
+        } else {
+            LOG(error) << "Could not find region info for region id '" << regionId << "' for shmId '" << shmId.shmId << "'";
+            return false;
+        }
+    } catch (bie&) {
+        LOG(error) << "Could not find management segment for shmid '" << shmId.shmId << "'";
+        return false;
+    }
+
+    return true;
+}
+bool Monitor::RegionIsPresent(const SessionId& sessionId, uint16_t regionId)
+{
+    ShmId shmId{makeShmIdStr(sessionId.sessionId)};
+    return RegionIsPresent(shmId, regionId);
+}
+
 void Monitor::PrintHelp()
 {
     LOG(info) << "controls: [x] close memory, "
@@ -572,7 +663,7 @@ std::vector<std::pair<std::string, bool>> Monitor::Cleanup(const ShmId& shmIdT,
         LOG(info) << "Cleaning up for shared memory id '" << shmId << "'...";
     }
 
-    string managementSegmentName("fmq_" + shmId + "_mng");
+    string managementSegmentName = MakeShmName(shmId, "mng");
     try {
         bipc::managed_shared_memory managementSegment(bipc::open_read_only, managementSegmentName.c_str());
 
@@ -587,14 +678,15 @@ std::vector<std::pair<std::string, bool>> Monitor::Cleanup(const ShmId& shmIdT,
                 string path = info.fPath.c_str();
                 int flags = info.fCreationFlags;
                 if (verbose) {
-                    LOG(info) << "Found RegionInfo with path: '" << path << "', flags: " << flags << ", fDestroyed: " << info.fDestroyed << ".";
+                    LOG(info) << "Found UnmanagedRegion with id: " << id << ", path: '" << path << "', flags: " << flags << ", fDestroyed: " << info.fDestroyed << ".";
                 }
                 if (!path.empty()) {
-                    result.emplace_back(Remove<bipc::file_mapping>(path + "fmq_" + shmId + "_rg_" + to_string(id), verbose));
+                    result.emplace_back(Remove<bipc::file_mapping>(path + MakeShmName(shmId, "rg", id), verbose));
                 } else {
-                    result.emplace_back(Remove<bipc::shared_memory_object>("fmq_" + shmId + "_rg_" + to_string(id), verbose));
+                    result.emplace_back(Remove<bipc::shared_memory_object>(MakeShmName(shmId, "rg", id), verbose));
                 }
-                result.emplace_back(Remove<bipc::message_queue>("fmq_" + shmId + "_rgq_" + to_string(id), verbose));
+                result.emplace_back(Remove<bipc::message_queue>(MakeShmName(shmId, "rgq", id), verbose));
+                result.emplace_back(Remove<bipc::shared_memory_object>(MakeShmName(shmId, "rrc", id), verbose));
             }
         }
 
@@ -604,7 +696,7 @@ std::vector<std::pair<std::string, bool>> Monitor::Cleanup(const ShmId& shmIdT,
                 LOG(info) << "Found " << shmSegments->size() << " managed segments...";
             }
             for (const auto& segment : *shmSegments) {
-                result.emplace_back(Remove<bipc::shared_memory_object>("fmq_" + shmId + "_m_" + to_string(segment.first), verbose));
+                result.emplace_back(Remove<bipc::shared_memory_object>(MakeShmName(shmId, "m", segment.first), verbose));
             }
         } else {
             if (verbose) {
@@ -634,7 +726,7 @@ std::vector<std::pair<std::string, bool>> Monitor::Cleanup(const SessionId& sess
 std::vector<std::pair<std::string, bool>> Monitor::CleanupFull(const ShmId& shmId, bool verbose /* = true */)
 {
     auto result = Cleanup(shmId, verbose);
-    result.emplace_back(Remove<bipc::named_mutex>("fmq_" + shmId.shmId + "_ms", verbose));
+    result.emplace_back(Remove<bipc::named_mutex>(MakeShmName(shmId.shmId, "ms"), verbose));
     return result;
 }
 
@@ -654,7 +746,7 @@ void Monitor::ResetContent(const ShmId& shmIdT, bool verbose /* = true */)
         cout << "Resetting segments content for shared memory id '" << shmId << "'..." << endl;
     }
 
-    string managementSegmentName("fmq_" + shmId + "_mng");
+    string managementSegmentName = MakeShmName(shmId, "mng");
     try {
         using namespace boost::interprocess;
         managed_shared_memory managementSegment(open_only, managementSegmentName.c_str());
@@ -662,18 +754,18 @@ void Monitor::ResetContent(const ShmId& shmIdT, bool verbose /* = true */)
         Uint16SegmentInfoHashMap* segmentInfos = managementSegment.find<Uint16SegmentInfoHashMap>(unique_instance).first;
         if (segmentInfos) {
             cout << "Found info for " << segmentInfos->size() << " managed segments" << endl;
-            for (const auto& s : *segmentInfos) {
+            for (const auto& [id, info] : *segmentInfos) {
                 if (verbose) {
-                    cout << "Resetting content of segment '" << "fmq_" << shmId << "_m_" << s.first << "'..." << endl;
+                    cout << "Resetting content of segment '" << MakeShmName(shmId, "m", id) << "'..." << endl;
                 }
                 try {
-                    if (s.second.fAllocationAlgorithm == AllocationAlgorithm::rbtree_best_fit) {
-                        RBTreeBestFitSegment segment(open_only, std::string("fmq_" + shmId + "_m_" + to_string(s.first)).c_str());
+                    if (info.fAllocationAlgorithm == AllocationAlgorithm::rbtree_best_fit) {
+                        RBTreeBestFitSegment segment(open_only, MakeShmName(shmId, "m", id).c_str());
                         void* ptr = segment.get_segment_manager();
                         size_t size = segment.get_segment_manager()->get_size();
                         new(ptr) segment_manager<char, rbtree_best_fit<mutex_family, offset_ptr<void>>, null_index>(size);
                     } else {
-                        SimpleSeqFitSegment segment(open_only, std::string("fmq_" + shmId + "_m_" + to_string(s.first)).c_str());
+                        SimpleSeqFitSegment segment(open_only, MakeShmName(shmId, "m", id).c_str());
                         void* ptr = segment.get_segment_manager();
                         size_t size = segment.get_segment_manager()->get_size();
                         new(ptr) segment_manager<char, simple_seq_fit<mutex_family, offset_ptr<void>>, null_index>(size);
@@ -683,7 +775,7 @@ void Monitor::ResetContent(const ShmId& shmIdT, bool verbose /* = true */)
                     }
                 } catch (bie& e) {
                     if (verbose) {
-                        cout << "Error resetting content of segment '" << std::string("fmq_" + shmId + "_m_" + to_string(s.first)) << "': " << e.what() << endl;
+                        cout << "Error resetting content of segment '" << MakeShmName(shmId, "m", id) << "': " << e.what() << endl;
                     }
                 }
             }
@@ -695,7 +787,8 @@ void Monitor::ResetContent(const ShmId& shmIdT, bool verbose /* = true */)
         if (shmRegions) {
             for (const auto& region : *shmRegions) {
                 uint16_t id = region.first;
-                Remove<bipc::message_queue>("fmq_" + shmId + "_rgq_" + to_string(id), verbose);
+                Remove<bipc::message_queue>(MakeShmName(shmId, "rgq", id), verbose);
+                Remove<bipc::shared_memory_object>(MakeShmName(shmId, "rrc", id), verbose);
             }
         }
     } catch (bie& e) {
@@ -724,7 +817,7 @@ void Monitor::ResetContent(const ShmId& shmIdT, const std::vector<SegmentConfig>
     using namespace boost::interprocess;
 
     std::string shmId = shmIdT.shmId;
-    std::string managementSegmentName("fmq_" + shmId + "_mng");
+    std::string managementSegmentName = MakeShmName(shmId, "mng");
     // delete management segment
     cout << "deleting management segment" << endl;
     Remove<bipc::shared_memory_object>(managementSegmentName, verbose);
@@ -772,7 +865,7 @@ Monitor::~Monitor()
         Cleanup(ShmId{fShmId});
     }
     if (!fViewOnly) {
-        RemoveMutex("fmq_" + fShmId + "_ms");
+        RemoveMutex(MakeShmName(fShmId, "ms"));
     }
 }
 

fairmq/shmem/Monitor.h

@@ -119,7 +119,7 @@ class Monitor
     /// @param sessionId session id
     static std::unordered_map<uint16_t, std::vector<BufferDebugInfo>> GetDebugInfo(const SessionId& sessionId);
     /// @brief Returns the amount of free memory in the specified segment
-    /// @param sessionId shmem id
+    /// @param shmId shmem id
     /// @param segmentId segment id
     /// @throws MonitorError
     static unsigned long GetFreeMemory(const ShmId& shmId, uint16_t segmentId);
@@ -128,6 +128,23 @@ class Monitor
     /// @param segmentId segment id
     /// @throws MonitorError
     static unsigned long GetFreeMemory(const SessionId& sessionId, uint16_t segmentId);
+    /// @brief Checks if a given segment can be opened
+    /// @param shmId shmem id
+    /// @param segmentId segment id
+    static bool SegmentIsPresent(const ShmId& shmId, uint16_t segmentId);
+    /// @brief Checks if a given segment can be opened
+    /// @param sessionId session id
+    /// @param segmentId segment id
+    static bool SegmentIsPresent(const SessionId& sessionId, uint16_t segmentId);
+    /// @brief Checks if a given region can be opened
+    /// @param shmId shmem id
+    /// @param regionId region id
+    static bool RegionIsPresent(const ShmId& shmId, uint16_t regionId);
+    /// @brief Checks if a given region can be opened
+    /// @param sessionId session id
+    /// @param regionId region id
+    static bool RegionIsPresent(const SessionId& sessionId, uint16_t regionId);
+
 
     static bool PrintShm(const ShmId& shmId);
     static void ListAll(const std::string& path);