From 03ba9eb5584bce6fdfa407387c58012883fbc7d7 Mon Sep 17 00:00:00 2001 From: Alexey Rybalchenko Date: Thu, 15 Jul 2021 23:54:46 +0200 Subject: [PATCH] Add --shm-zero-segment-on-creation option --- fairmq/plugins/config/Config.cxx | 5 ++-- fairmq/shmem/Manager.h | 39 ++++++++++++++++++++------------ test/transport/_options.cxx | 22 ++++++++++++++++++ 3 files changed, 50 insertions(+), 16 deletions(-) diff --git a/fairmq/plugins/config/Config.cxx b/fairmq/plugins/config/Config.cxx index 0c4a1218..4226b288 100644 --- a/fairmq/plugins/config/Config.cxx +++ b/fairmq/plugins/config/Config.cxx @@ -67,9 +67,10 @@ Plugin::ProgOptions ConfigPluginProgramOptions() ("shm-segment-size", po::value()->default_value(2ULL << 30), "Shared memory: size of the shared memory segment (in bytes).") ("shm-allocation", po::value()->default_value("rbtree_best_fit"), "Shared memory allocation algorithm: rbtree_best_fit/simple_seq_fit.") ("shm-segment-id", po::value()->default_value(0), "EXPERIMENTAL: Shared memory segment id for message creation.") - ("shm-mlock-segment", po::value()->default_value(false), "Shared memory: mlock the shared memory segment after initialization.") + ("shm-mlock-segment", po::value()->default_value(false), "Shared memory: mlock the shared memory segment after initialization (opened or created).") ("shm-mlock-segment-on-creation", po::value()->default_value(false), "Shared memory: mlock the shared memory segment only once when created.") - ("shm-zero-segment", po::value()->default_value(false), "Shared memory: zero the shared memory segment memory after initialization.") + ("shm-zero-segment", po::value()->default_value(false), "Shared memory: zero the shared memory segment memory after initialization (opened or created).") + ("shm-zero-segment-on-creation", po::value()->default_value(false), "Shared memory: zero the shared memory segment memory only once when created.") ("shm-throw-bad-alloc", po::value()->default_value(true), "Throw a fair::mq::MessageBadAlloc if cannot allocate a message (retry if false).") ("shm-monitor", po::value()->default_value(true), "Shared memory: run monitor daemon.") ("shm-no-cleanup", po::value()->default_value(false), "Shared memory: do not cleanup the memory when last device leaves.") diff --git a/fairmq/shmem/Manager.h b/fairmq/shmem/Manager.h index 9be47755..42d6113a 100644 --- a/fairmq/shmem/Manager.h +++ b/fairmq/shmem/Manager.h @@ -92,12 +92,14 @@ class Manager bool mlockSegment = false; bool mlockSegmentOnCreation = false; bool zeroSegment = false; + bool zeroSegmentOnCreation = false; bool autolaunchMonitor = false; std::string allocationAlgorithm("rbtree_best_fit"); if (config) { mlockSegment = config->GetProperty("shm-mlock-segment", mlockSegment); mlockSegmentOnCreation = config->GetProperty("shm-mlock-segment-on-creation", mlockSegmentOnCreation); zeroSegment = config->GetProperty("shm-zero-segment", zeroSegment); + zeroSegmentOnCreation = config->GetProperty("shm-zero-segment-on-creation", zeroSegmentOnCreation); autolaunchMonitor = config->GetProperty("shm-monitor", autolaunchMonitor); allocationAlgorithm = config->GetProperty("shm-allocation", allocationAlgorithm); } else { @@ -165,11 +167,10 @@ class Manager ss << "Created "; (fEventCounter->fCount)++; if (mlockSegmentOnCreation) { - LOG(error) << "Locking the managed segment memory pages..."; - if (mlock(boost::apply_visitor(SegmentAddress(), fSegments.at(fSegmentId)), boost::apply_visitor(SegmentSize(), fSegments.at(fSegmentId))) == -1) { - LOG(error) << "Could not lock the managed segment memory. Code: " << errno << ", reason: " << strerror(errno); - } - LOG(error) << "Successfully locked the managed segment memory pages."; + MlockSegment(fSegmentId); + } + if (zeroSegmentOnCreation) { + ZeroSegment(fSegmentId); } } else { op = "open"; @@ -200,17 +201,10 @@ class Manager } if (mlockSegment) { - LOG(debug) << "Locking the managed segment memory pages..."; - if (mlock(boost::apply_visitor(SegmentAddress(), fSegments.at(fSegmentId)), boost::apply_visitor(SegmentSize(), fSegments.at(fSegmentId))) == -1) { - LOG(error) << "Could not lock the managed segment memory. Code: " << errno << ", reason: " << strerror(errno); - throw TransportError(tools::ToString("Could not lock the managed segment memory: ", strerror(errno))); - } - LOG(debug) << "Successfully locked the managed segment memory pages."; + MlockSegment(fSegmentId); } if (zeroSegment) { - LOG(debug) << "Zeroing the managed segment free memory..."; - boost::apply_visitor(SegmentMemoryZeroer(), fSegments.at(fSegmentId)); - LOG(debug) << "Successfully zeroed the managed segment free memory."; + ZeroSegment(fSegmentId); } #ifdef FAIRMQ_DEBUG_MODE @@ -229,6 +223,23 @@ class Manager Manager(const Manager&) = delete; Manager operator=(const Manager&) = delete; + void ZeroSegment(uint16_t id) + { + LOG(debug) << "Zeroing the managed segment free memory..."; + boost::apply_visitor(SegmentMemoryZeroer(), fSegments.at(id)); + LOG(debug) << "Successfully zeroed the managed segment free memory."; + } + + void MlockSegment(uint16_t id) + { + LOG(debug) << "Locking the managed segment memory pages..."; + if (mlock(boost::apply_visitor(SegmentAddress(), fSegments.at(id)), boost::apply_visitor(SegmentSize(), fSegments.at(id))) == -1) { + LOG(error) << "Could not lock the managed segment memory. Code: " << errno << ", reason: " << strerror(errno); + throw TransportError(tools::ToString("Could not lock the managed segment memory: ", strerror(errno))); + } + LOG(debug) << "Successfully locked the managed segment memory pages."; + } + static void StartMonitor(const std::string& id) { using namespace boost::interprocess; diff --git a/test/transport/_options.cxx b/test/transport/_options.cxx index c8df2d68..e603ebec 100644 --- a/test/transport/_options.cxx +++ b/test/transport/_options.cxx @@ -75,7 +75,24 @@ void ZeroingAndMlock(const string& transport) config.SetProperty("shm-segment-size", 16384); config.SetProperty("shm-zero-segment", true); config.SetProperty("shm-mlock-segment", true); + + auto factory = FairMQTransportFactory::CreateTransportFactory(transport, fair::mq::tools::Uuid(), &config); + + FairMQMessagePtr outMsg(factory->CreateMessage(10000)); + char test[10000]; + memset(test, 0, sizeof(test)); + ASSERT_EQ(memcmp(test, outMsg->GetData(), outMsg->GetSize()), 0); +} + +void ZeroingAndMlockOnCreation(const string& transport) +{ + size_t session{fair::mq::tools::UuidHash()}; + + fair::mq::ProgOptions config; + config.SetProperty("session", to_string(session)); + config.SetProperty("shm-segment-size", 16384); config.SetProperty("shm-mlock-segment-on-creation", true); + config.SetProperty("shm-zero-segment-on-creation", true); auto factory = FairMQTransportFactory::CreateTransportFactory(transport, fair::mq::tools::Uuid(), &config); @@ -100,4 +117,9 @@ TEST(ZeroingAndMlock, shmem) ZeroingAndMlock("shmem"); } +TEST(ZeroingAndMlockOnCreation, shmem) +{ + ZeroingAndMlockOnCreation("shmem"); +} + } // namespace