Mirror/FairMQ
1
0
Fork 0
mirror of https://github.com/FairRootGroup/FairMQ.git synced 2025-10-15 17:41:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement bulk callbacks for unmanaged regions

Browse Source
This commit is contained in:
Alexey Rybalchenko
2020-05-15 00:20:49 +02:00
parent a15d59c725
commit d22023bcb5
24 changed files with 243 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
fairmq/shmem/Manager.cxx
View File

@@ -101,7 +101,12 @@ void Manager::StartMonitor(const string& id)
}
}
pair<bipc::mapped_region*, uint64_t> Manager::CreateRegion(const size_t size, const int64_t userFlags, RegionCallback callback, const string& path /* = "" */, int flags /* = 0 */)
pair<bipc::mapped_region*, uint64_t> Manager::CreateRegion(const size_t size,
const int64_t userFlags,
RegionCallback callback,
RegionBulkCallback bulkCallback,
const string& path /* = "" */,
int flags /* = 0 */)
{
try {
@@ -134,7 +139,7 @@ pair<bipc::mapped_region*, uint64_t> Manager::CreateRegion(const size_t size, co
// create region info
fRegionInfos->emplace(id, RegionInfo(path.c_str(), flags, userFlags, fShmVoidAlloc));
auto r = fRegions.emplace(id, tools::make_unique<Region>(*this, id, size, false, callback, path, flags));
auto r = fRegions.emplace(id, tools::make_unique<Region>(*this, id, size, false, callback, bulkCallback, path, flags));
// LOG(debug) << "Created region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
r.first->second->StartReceivingAcks();
@@ -182,7 +187,7 @@ Region* Manager::GetRegionUnsafe(const uint64_t id)
int flags = regionInfo.fFlags;
// LOG(debug) << "Located remote region with id '" << id << "', path: '" << path << "', flags: '" << flags << "'";
auto r = fRegions.emplace(id, tools::make_unique<Region>(*this, id, 0, true, nullptr, path, flags));
auto r = fRegions.emplace(id, tools::make_unique<Region>(*this, id, 0, true, nullptr, nullptr, path, flags));
return r.first->second.get();
} catch (bie& e) {
LOG(warn) << "Could not get remote region for id: " << id;

7
fairmq/shmem/Manager.h
View File

@@ -70,7 +70,12 @@ class Manager
int IncrementDeviceCounter();
int DecrementDeviceCounter();
std::pair<boost::interprocess::mapped_region*, uint64_t> CreateRegion(const size_t size, const int64_t userFlags, RegionCallback callback, const std::string& path = "", int flags = 0);
std::pair<boost::interprocess::mapped_region*, uint64_t> CreateRegion(const size_t size,
const int64_t userFlags,
RegionCallback callback,
RegionBulkCallback bulkCallback,
const std::string& path = "",
int flags = 0);
Region* GetRegion(const uint64_t id);
Region* GetRegionUnsafe(const uint64_t id);
void RemoveRegion(const uint64_t id);

17
fairmq/shmem/Region.cxx
View File

@@ -33,7 +33,7 @@ namespace mq
namespace shmem
{
Region::Region(Manager& manager, uint64_t id, uint64_t size, bool remote, RegionCallback callback, const string& path /* = "" */, int flags /* = 0 */)
Region::Region(Manager& manager, uint64_t id, uint64_t size, bool remote, RegionCallback callback, RegionBulkCallback bulkCallback, const string& path, int flags)
: fManager(manager)
, fRemote(remote)
, fStop(false)
@@ -46,6 +46,7 @@ Region::Region(Manager& manager, uint64_t id, uint64_t size, bool remote, Region
, fReceiveAcksWorker()
, fSendAcksWorker()
, fCallback(callback)
, fBulkCallback(bulkCallback)
{
if (path != "") {
fName = string(path + fName);
@@ -110,14 +111,22 @@ void Region::ReceiveAcks()
unsigned int priority;
bipc::message_queue::size_type recvdSize;
unique_ptr<RegionBlock[]> blocks = tools::make_unique<RegionBlock[]>(fAckBunchSize);
std::vector<fair::mq::RegionBlock> result;
result.reserve(fAckBunchSize);
while (!fStop) { // end thread condition (should exist until region is destroyed)
auto rcvTill = bpt::microsec_clock::universal_time() + bpt::milliseconds(500);
while (fQueue->timed_receive(blocks.get(), fAckBunchSize * sizeof(RegionBlock), recvdSize, priority, rcvTill)) {
// LOG(debug) << "received: " << block.fHandle << " " << block.fSize << " " << block.fMessageId;
if (fCallback) {
const auto numBlocks = recvdSize / sizeof(RegionBlock);
const auto numBlocks = recvdSize / sizeof(RegionBlock);
if (fBulkCallback) {
result.clear();
for (size_t i = 0; i < numBlocks; i++) {
result.emplace_back(reinterpret_cast<char*>(fRegion.get_address()) + blocks[i].fHandle, blocks[i].fSize, reinterpret_cast<void*>(blocks[i].fHint));
}
fBulkCallback(result);
} else if (fCallback) {
for (size_t i = 0; i < numBlocks; i++) {
fCallback(reinterpret_cast<char*>(fRegion.get_address()) + blocks[i].fHandle, blocks[i].fSize, reinterpret_cast<void*>(blocks[i].fHint));
}
@@ -125,7 +134,7 @@ void Region::ReceiveAcks()
}
} // while !fStop
LOG(debug) << "receive ack worker for " << fName << " leaving.";
LOG(debug) << "ReceiveAcks() worker for " << fName << " leaving.";
}
void Region::ReleaseBlock(const RegionBlock &block)

3
fairmq/shmem/Region.h
View File

@@ -40,7 +40,7 @@ class Manager;
struct Region
{
Region(Manager& manager, uint64_t id, uint64_t size, bool remote, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0);
Region(Manager& manager, uint64_t id, uint64_t size, bool remote, RegionCallback callback, RegionBulkCallback bulkCallback, const std::string& path, int flags);
Region() = delete;
@@ -76,6 +76,7 @@ struct Region
std::thread fReceiveAcksWorker;
std::thread fSendAcksWorker;
RegionCallback fCallback;
RegionBulkCallback fBulkCallback;
};
} // namespace shmem

14
fairmq/shmem/TransportFactory.cxx
View File

@@ -161,12 +161,22 @@ PollerPtr TransportFactory::CreatePoller(const unordered_map<string, vector<Fair
UnmanagedRegionPtr TransportFactory::CreateUnmanagedRegion(const size_t size, RegionCallback callback, const std::string& path /* = "" */, int flags /* = 0 */)
{
return tools::make_unique<UnmanagedRegion>(*fManager, size, callback, path, flags, this);
return tools::make_unique<UnmanagedRegion>(*fManager, size, 0, callback, nullptr, path, flags, this);
}
UnmanagedRegionPtr TransportFactory::CreateUnmanagedRegion(const size_t size, RegionBulkCallback bulkCallback, const std::string& path /* = "" */, int flags /* = 0 */)
{
return tools::make_unique<UnmanagedRegion>(*fManager, size, 0, nullptr, bulkCallback, path, flags, this);
}
UnmanagedRegionPtr TransportFactory::CreateUnmanagedRegion(const size_t size, const int64_t userFlags, RegionCallback callback, const std::string& path /* = "" */, int flags /* = 0 */)
{
return tools::make_unique<UnmanagedRegion>(*fManager, size, userFlags, callback, path, flags, this);
return tools::make_unique<UnmanagedRegion>(*fManager, size, userFlags, callback, nullptr, path, flags, this);
}
UnmanagedRegionPtr TransportFactory::CreateUnmanagedRegion(const size_t size, const int64_t userFlags, RegionBulkCallback bulkCallback, const std::string& path /* = "" */, int flags /* = 0 */)
{
return tools::make_unique<UnmanagedRegion>(*fManager, size, userFlags, nullptr, bulkCallback, path, flags, this);
}
void TransportFactory::SubscribeToRegionEvents(RegionEventCallback callback)

2
fairmq/shmem/TransportFactory.h
View File

@@ -50,7 +50,9 @@ class TransportFactory final : public fair::mq::TransportFactory
PollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const override;
UnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0) override;
UnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, RegionBulkCallback callback = nullptr, const std::string& path = "", int flags = 0) override;
UnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, int64_t userFlags, RegionCallback callback = nullptr, const std::string& path = "", int flags = 0) override;
UnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, int64_t userFlags, RegionBulkCallback callback = nullptr, const std::string& path = "", int flags = 0) override;
void SubscribeToRegionEvents(RegionEventCallback callback) override;
bool SubscribedToRegionEvents() override;

15
fairmq/shmem/UnmanagedRegion.h
View File

@@ -36,17 +36,20 @@ class UnmanagedRegion final : public fair::mq::UnmanagedRegion
friend class Socket;
public:
UnmanagedRegion(Manager& manager, const size_t size, RegionCallback callback, const std::string& path = "", int flags = 0, FairMQTransportFactory* factory = nullptr)
: UnmanagedRegion(manager, size, 0, callback, path, flags, factory)
{}
UnmanagedRegion(Manager& manager, const size_t size, const int64_t userFlags, RegionCallback callback, const std::string& path = "", int flags = 0, FairMQTransportFactory* factory = nullptr)
UnmanagedRegion(Manager& manager,
const size_t size,
const int64_t userFlags,
RegionCallback callback,
RegionBulkCallback bulkCallback,
const std::string& path = "",
int flags = 0,
FairMQTransportFactory* factory = nullptr)
: FairMQUnmanagedRegion(factory)
, fManager(manager)
, fRegion(nullptr)
, fRegionId(0)
{
auto result = fManager.CreateRegion(size, userFlags, callback, path, flags);
auto result = fManager.CreateRegion(size, userFlags, callback, bulkCallback, path, flags);
fRegion = result.first;
fRegionId = result.second;
}