test: support an alternative runtime library dir per test

- introduce FAIRMQ_TEST_LD_LIBRARY_PATH, which prepends a directory to
  each test's environment via ctest, so the tests can run against an
  alternative runtime library (e.g. a tsan-instrumented libstdc++)
- LD_LIBRARY_PATH rather than an injected rpath: an rpath added via the
  linker flags cannot precede the rpath spack's gcc adds through its
  specs file, so the compiler's own libstdc++ would keep winning the
  runtime search order
- scoped per test on purpose: an instrumented library has unresolved
  __tsan_* symbols and must not be loaded into uninstrumented tools
  like cmake, ctest or ninja
- fail the configuration instead of silently dropping the injection on
  CMake < 3.22 (ENVIRONMENT_MODIFICATION)
- cover the example tests too; they share the instrumented runtime but
  not the locale-cache warmup (their main() is the installed public
  header). The custom-controller env block was dead before: it tested
  lsan_options, which only ever existed in the add_example() function
  scope, so the test also never received the LSan suppressions

CMakeLists.txt

@@ -46,6 +46,13 @@ include(FairMQDependencies)
 
 
 # Targets ######################################################################
+if(FAIRMQ_TEST_LD_LIBRARY_PATH AND CMAKE_VERSION VERSION_LESS 3.22)
+  # The per-test injection relies on ctest's ENVIRONMENT_MODIFICATION, which
+  # older CMake silently drops -- the tests would run against the default
+  # runtime while looking green.
+  message(FATAL_ERROR "FAIRMQ_TEST_LD_LIBRARY_PATH requires CMake >= 3.22")
+endif()
+
 if(BUILD_FAIRMQ)
   add_subdirectory(fairmq)
 endif()

examples/CMakeLists.txt

@@ -12,6 +12,22 @@ set(test_script_prefix "test-ex")
 set(testsuite "Example")
 set(transports "zeromq" "shmem")
 
+# Environment for every example test (directory scope, so the subdirectories
+# see it too)
+set(env_mods)
+if(ENABLE_SANITIZER_LEAK AND CMAKE_VERSION VERSION_GREATER_EQUAL 3.22)
+  get_filename_component(lsan_supps "${CMAKE_SOURCE_DIR}/test/leak_sanitizer_suppressions.txt" ABSOLUTE)
+  list(APPEND env_mods "LSAN_OPTIONS=set:suppressions=${lsan_supps}")
+endif()
+# Run the example tests against an alternative runtime library directory as
+# well (see test/CMakeLists.txt). The test scripts only launch instrumented
+# binaries (bash itself does not link libstdc++). Note: unlike the
+# testsuites, the example binaries get no locale-cache warmup -- their
+# main() comes from the installed public header fairmq/runDevice.h.
+if(FAIRMQ_TEST_LD_LIBRARY_PATH)
+  list(APPEND env_mods "LD_LIBRARY_PATH=path_list_prepend:${FAIRMQ_TEST_LD_LIBRARY_PATH}")
+endif()
+
 function(add_example)
   cmake_parse_arguments(PARSE_ARGV 0 ARG
     "CONFIG;NO_TRANSPORT;NO_TEST"
@@ -29,11 +45,6 @@ function(add_example)
     message(FATAL_ERROR "NAME arg is required")
   endif()
 
-  if(ENABLE_SANITIZER_LEAK AND CMAKE_VERSION VERSION_GREATER_EQUAL 3.22)
-    get_filename_component(lsan_supps "${CMAKE_SOURCE_DIR}/test/leak_sanitizer_suppressions.txt" ABSOLUTE)
-    set(lsan_options "LSAN_OPTIONS=set:suppressions=${lsan_supps}")
-  endif()
-
   if(ARG_DEVICE)
     set(exe_targets)
     foreach(device IN LISTS ARG_DEVICE)
@@ -78,8 +89,8 @@ function(add_example)
       set(test "${testsuite}.${name}.${transport}")
       add_test(NAME ${test} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${test_script} ${transport})
       set_tests_properties(${test} PROPERTIES TIMEOUT "30")
-      if(lsan_options)
+      if(env_mods)
-        set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION ${lsan_options})
+        set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION "${env_mods}")
       endif()
     else()
       foreach(transport IN LISTS transports)
@@ -88,16 +99,16 @@ function(add_example)
             set(test "${testsuite}.${name}.${variant}.${transport}")
             add_test(NAME ${test} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${test_script} ${transport} ${variant})
             set_tests_properties(${test} PROPERTIES TIMEOUT "30")
-            if(lsan_options)
+            if(env_mods)
-              set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION ${lsan_options})
+              set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION "${env_mods}")
             endif()
           endforeach()
         else()
           set(test "${testsuite}.${name}.${transport}")
           add_test(NAME ${test} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${test_script} ${transport})
           set_tests_properties(${test} PROPERTIES TIMEOUT "30")
-          if(lsan_options)
+          if(env_mods)
-            set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION ${lsan_options})
+            set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION "${env_mods}")
           endif()
         endif()
       endforeach()

examples/custom-controller/CMakeLists.txt

@@ -15,6 +15,8 @@ set_target_properties(${exe} PROPERTIES ENABLE_EXPORTS ON)
 set(test "${testsuite}.${name}")
 add_test(NAME ${test} COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${exe})
 set_tests_properties(${test} PROPERTIES TIMEOUT 30)
-if(lsan_options)
+# (the previous `if(lsan_options)` here could never fire -- that variable
-  set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION ${lsan_options})
+# only ever existed in the add_example() function scope)
+if(env_mods)
+  set_tests_properties(${test} PROPERTIES ENVIRONMENT_MODIFICATION "${env_mods}")
 endif()

test/CMakeLists.txt

@@ -24,9 +24,19 @@ if(definitions)
   set(definitions DEFINITIONS ${definitions})
 endif()
 
+set(test_environment)
 if(ENABLE_SANITIZER_LEAK)
   get_filename_component(lsan_supps "${CMAKE_CURRENT_SOURCE_DIR}/leak_sanitizer_suppressions.txt" ABSOLUTE)
-  set(environment ENVIRONMENT "LSAN_OPTIONS=set:suppressions=${lsan_supps}")
+  list(APPEND test_environment "LSAN_OPTIONS=set:suppressions=${lsan_supps}")
+endif()
+# Run the tests against an alternative runtime library directory, e.g. a
+# tsan-instrumented libstdc++. Set per test (not at the job level), because
+# such a library must only be loaded into instrumented executables.
+if(FAIRMQ_TEST_LD_LIBRARY_PATH)
+  list(APPEND test_environment "LD_LIBRARY_PATH=path_list_prepend:${FAIRMQ_TEST_LD_LIBRARY_PATH}")
+endif()
+if(test_environment)
+  set(environment ENVIRONMENT ${test_environment})
 endif()
 
 add_testhelper(runTestDevice