Mirror/FairMQ
1
0
Fork 0
mirror of https://github.com/FairRootGroup/FairMQ.git synced 2025-10-12 16:21:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor the transport interface

Browse Source 
- give transport Initialize() method with access to device config.
 - avoid using global context in the transport.
 - simplify shutdown procedure (no need for extra thread).
This commit is contained in:
Alexey Rybalchenko 2017-04-04 17:14:37 +02:00 committed by Mohammad Al-Turany
parent d7eb692951
commit 5aaf27bf02
25 changed files with 278 additions and 358 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
fairmq/CMakeLists.txt
View File

@ -77,14 +77,12 @@ set(FAIRMQ_HEADER_FILES
options/FairProgOptions.h
options/FairProgOptionsHelper.h
runFairMQDevice.h
shmem/FairMQContextSHM.h
shmem/FairMQMessageSHM.h
shmem/FairMQPollerSHM.h
shmem/FairMQSocketSHM.h
shmem/FairMQTransportFactorySHM.h
tools/FairMQTools.h
tools/runSimpleMQStateMachine.h
zeromq/FairMQContextZMQ.h
zeromq/FairMQMessageZMQ.h
zeromq/FairMQPollerZMQ.h
zeromq/FairMQSocketZMQ.h
@ -128,12 +126,10 @@ set(FAIRMQ_SOURCE_FILES
options/FairMQProgOptions.cxx
options/FairMQSuboptParser.cxx
options/FairProgOptions.cxx
shmem/FairMQContextSHM.cxx
shmem/FairMQMessageSHM.cxx
shmem/FairMQPollerSHM.cxx
shmem/FairMQSocketSHM.cxx
shmem/FairMQTransportFactorySHM.cxx
zeromq/FairMQContextZMQ.cxx
zeromq/FairMQMessageZMQ.cxx
zeromq/FairMQPollerZMQ.cxx
zeromq/FairMQSocketZMQ.cxx
@ -176,7 +172,6 @@ target_include_directories(FairMQ
$<INSTALL_INTERFACE:include/fairmq>
)
##################
# link libraries #
##################

4
fairmq/FairMQChannel.cxx
View File

@ -581,9 +581,9 @@ void FairMQChannel::InitTransport(shared_ptr<FairMQTransportFactory> factory)
fTransportType = factory->GetType();
}
bool FairMQChannel::InitCommandInterface(int numIoThreads)
bool FairMQChannel::InitCommandInterface()
{
fChannelCmdSocket = fTransportFactory->CreateSocket("sub", "device-commands", numIoThreads, "internal");
fChannelCmdSocket = fTransportFactory->CreateSocket("sub", "device-commands", "internal");
if (fChannelCmdSocket)
{
fChannelCmdSocket->Connect("inproc://commands");

2
fairmq/FairMQChannel.h
View File

@ -258,7 +258,7 @@ class FairMQChannel
bool CheckCompatibility(std::vector<std::unique_ptr<FairMQMessage>>& msgVec) const;
void InitTransport(std::shared_ptr<FairMQTransportFactory> factory);
bool InitCommandInterface(int numIoThreads);
bool InitCommandInterface();
bool HandleUnblock() const;

81
fairmq/FairMQDevice.cxx
View File

@ -128,7 +128,7 @@ void FairMQDevice::AttachChannels(list<FairMQChannel*>& chans)
{
if (AttachChannel(**itr))
{
(*itr)->InitCommandInterface(fNumIoThreads);
(*itr)->InitCommandInterface();
chans.erase(itr++);
}
else
@ -154,7 +154,7 @@ void FairMQDevice::InitWrapper()
if (fDeviceCmdSockets.empty())
{
auto p = fDeviceCmdSockets.emplace(fTransportFactory->GetType(), fTransportFactory->CreateSocket("pub", "device-commands", fNumIoThreads, fId));
auto p = fDeviceCmdSockets.emplace(fTransportFactory->GetType(), fTransportFactory->CreateSocket("pub", "device-commands", fId));
if (p.second)
{
p.first->second->Bind("inproc://commands");
@ -282,7 +282,7 @@ bool FairMQDevice::AttachChannel(FairMQChannel& ch)
//(re-)init socket
if (!ch.fSocket)
{
ch.fSocket = ch.fTransportFactory->CreateSocket(ch.fType, ch.fName, fNumIoThreads, fId);
ch.fSocket = ch.fTransportFactory->CreateSocket(ch.fType, ch.fName, fId);
}
// set high water marks
@ -837,6 +837,7 @@ void FairMQDevice::SetTransport(FairMQTransportFactory* factory)
{
fTransportFactory = shared_ptr<FairMQTransportFactory>(factory);
pair<FairMQ::Transport, shared_ptr<FairMQTransportFactory>> t(fTransportFactory->GetType(), fTransportFactory);
fTransportFactory->Initialize(fConfig);
fTransports.insert(t);
}
else
@ -870,9 +871,7 @@ shared_ptr<FairMQTransportFactory> FairMQDevice::AddTransport(const string& tran
#endif
else
{
LOG(ERROR) << "Unavailable transport requested: "
<< "\"" << transport << "\""
<< ". Available are: "
LOG(ERROR) << "Unavailable transport requested: " << "\"" << transport << "\"" << ". Available are: "
<< "\"zeromq\""
<< "\"shmem\""
#ifdef NANOMSG_FOUND
@ -885,9 +884,10 @@ shared_ptr<FairMQTransportFactory> FairMQDevice::AddTransport(const string& tran
LOG(DEBUG) << "Adding '" << transport << "' transport to the device.";
pair<FairMQ::Transport, shared_ptr<FairMQTransportFactory>> trPair(FairMQ::TransportTypes.at(transport), tr);
tr->Initialize(fConfig);
fTransports.insert(trPair);
auto p = fDeviceCmdSockets.emplace(tr->GetType(), tr->CreateSocket("pub", "device-commands", fNumIoThreads, fId));
auto p = fDeviceCmdSockets.emplace(tr->GetType(), tr->CreateSocket("pub", "device-commands", fId));
if (p.second)
{
p.first->second->Bind("inproc://commands");
@ -997,6 +997,8 @@ void FairMQDevice::LogSocketRates()
t0 = get_timestamp();
LOG(DEBUG) << "<channel>: in: <#msgs> (<MB>) out: <#msgs> (<MB>)";
while (CheckCurrentState(RUNNING))
{
t1 = get_timestamp();
@ -1014,24 +1016,23 @@ void FairMQDevice::LogSocketRates()
intervalCounters.at(i) = 0;
bytesInNew.at(i) = vi->GetBytesRx();
mbPerSecIn.at(i) = (static_cast<double>(bytesInNew.at(i) - bytesIn.at(i)) / (1000. * 1000.)) / static_cast<double>(msSinceLastLog) * 1000.;
bytesIn.at(i) = bytesInNew.at(i);
msgInNew.at(i) = vi->GetMessagesRx();
msgPerSecIn.at(i) = static_cast<double>(msgInNew.at(i) - msgIn.at(i)) / static_cast<double>(msSinceLastLog) * 1000.;
msgIn.at(i) = msgInNew.at(i);
bytesOutNew.at(i) = vi->GetBytesTx();
mbPerSecOut.at(i) = (static_cast<double>(bytesOutNew.at(i) - bytesOut.at(i)) / (1000. * 1000.)) / static_cast<double>(msSinceLastLog) * 1000.;
bytesOut.at(i) = bytesOutNew.at(i);
msgOutNew.at(i) = vi->GetMessagesTx();
mbPerSecIn.at(i) = (static_cast<double>(bytesInNew.at(i) - bytesIn.at(i)) / (1000. * 1000.)) / static_cast<double>(msSinceLastLog) * 1000.;
msgPerSecIn.at(i) = static_cast<double>(msgInNew.at(i) - msgIn.at(i)) / static_cast<double>(msSinceLastLog) * 1000.;
mbPerSecOut.at(i) = (static_cast<double>(bytesOutNew.at(i) - bytesOut.at(i)) / (1000. * 1000.)) / static_cast<double>(msSinceLastLog) * 1000.;
msgPerSecOut.at(i) = static_cast<double>(msgOutNew.at(i) - msgOut.at(i)) / static_cast<double>(msSinceLastLog) * 1000.;
bytesIn.at(i) = bytesInNew.at(i);
msgIn.at(i) = msgInNew.at(i);
bytesOut.at(i) = bytesOutNew.at(i);
msgOut.at(i) = msgOutNew.at(i);
LOG(DEBUG) << filteredChannelNames.at(i) << ": "
<< "in: " << msgPerSecIn.at(i) << " msg (" << mbPerSecIn.at(i) << " MB), "
<< "out: " << msgPerSecOut.at(i) << " msg (" << mbPerSecOut.at(i) << " MB)";
<< "in: " << msgPerSecIn.at(i) << " (" << mbPerSecIn.at(i) << " MB) "
<< "out: " << msgPerSecOut.at(i) << " (" << mbPerSecOut.at(i) << " MB)";
}
++i;
@ -1195,36 +1196,29 @@ bool FairMQDevice::Terminated()
return fTerminationRequested;
}
void FairMQDevice::Terminate()
void FairMQDevice::Exit()
{
// Termination signal has to be sent only once to any socket.
for (auto& kv : fDeviceCmdSockets)
// ask transports to terminate transfers
for (const auto& t : fTransports)
{
kv.second->Terminate();
}
// if (!fDeviceCmdSockets.empty())
// {
// fDeviceCmdSockets[0]->Terminate();
// }
t.second->Shutdown();
}
void FairMQDevice::Shutdown()
{
LOG(DEBUG) << "Closing sockets...";
// iterate over the channels map
for (const auto& mi : fChannels)
// iterate over the channels
for (const auto& c : fChannels)
{
// iterate over the channels vector
for (const auto& vi : mi.second)
// iterate over the sub-channels
for (const auto& sc : c.second)
{
if (vi.fSocket)
if (sc.fSocket)
{
vi.fSocket->Close();
sc.fSocket->Close();
}
if (vi.fChannelCmdSocket)
if (sc.fChannelCmdSocket)
{
vi.fChannelCmdSocket->Close();
sc.fChannelCmdSocket->Close();
}
}
}
@ -1234,12 +1228,15 @@ void FairMQDevice::Shutdown()
s.second->Close();
}
// if (!fDeviceCmdSockets.empty())
// {
// fDeviceCmdSockets[0]->Close();
// }
LOG(DEBUG) << "Closed all sockets!";
// ask transports to terminate
for (const auto& t : fTransports)
{
t.second->Terminate();
}
LOG(DEBUG) << "All transports exited.";
}
FairMQDevice::~FairMQDevice()

14
fairmq/FairMQDevice.h
View File

@ -337,7 +337,10 @@ class FairMQDevice : public FairMQStateMachine, public FairMQConfigurable
void SetTransport(const std::string& transport = "zeromq");
void SetConfig(FairMQProgOptions& config);
const FairMQProgOptions* GetConfig() const {return fConfig;}
const FairMQProgOptions* GetConfig() const
{
return fConfig;
}
/// Implements the sort algorithm used in SortChannel()
/// @param lhs Right hand side value for comparison
@ -442,20 +445,19 @@ class FairMQDevice : public FairMQStateMachine, public FairMQConfigurable
void ResetTaskWrapper();
/// Handles the Reset() method
void ResetWrapper();
/// Shuts down the device (closses socket connections)
void Shutdown();
/// Terminates the transport interface
void Terminate();
/// Unblocks blocking channel send/receive calls
void Unblock();
/// Shuts down the transports and the device
void Exit();
/// Attach (bind/connect) channels in the list
void AttachChannels(std::list<FairMQChannel*>& chans);
/// Sets up and connects/binds a socket to an endpoint
/// return a string with the actual endpoint if it happens
//to stray from default.
/// to stray from default.
bool ConnectEndpoint(FairMQSocket& socket, std::string& endpoint);
bool BindEndpoint(FairMQSocket& socket, std::string& endpoint);
/// Attaches the channel to all listed endpoints

1
fairmq/FairMQSocket.h
View File

@ -49,7 +49,6 @@ class FairMQSocket
virtual void* GetSocket() const = 0;
virtual int GetSocket(int nothing) const = 0;
virtual void Close() = 0;
virtual void Terminate() = 0;
virtual void Interrupt() = 0;
virtual void Resume() = 0;

9
fairmq/FairMQStateMachine.h
View File

@ -73,7 +73,6 @@ struct FairMQFSM_ : public msmf::state_machine_def<FairMQFSM_>
public:
FairMQFSM_()
: fWorkerThread()
, fTerminateStateThread()
, fWork()
, fWorkAvailableCondition()
, fWorkDoneCondition()
@ -329,9 +328,7 @@ struct FairMQFSM_ : public msmf::state_machine_def<FairMQFSM_>
fsm.fWorkerThread.join();
}
fsm.fTerminateStateThread = std::thread(&FairMQFSM_::Terminate, &fsm);
fsm.Shutdown();
fsm.fTerminateStateThread.join();
fsm.Exit();
}
};
@ -358,8 +355,7 @@ struct FairMQFSM_ : public msmf::state_machine_def<FairMQFSM_>
virtual void Reset() {}
virtual void ResetTaskWrapper() {}
virtual void ResetTask() {}
virtual void Shutdown() {}
virtual void Terminate() {} // Termination method called during StopFct action.
virtual void Exit() {}
virtual void Unblock() {} // Method to send commands.
void Worker()
@ -526,7 +522,6 @@ struct FairMQFSM_ : public msmf::state_machine_def<FairMQFSM_>
// this is to run certain functions in a separate thread
std::thread fWorkerThread;
std::thread fTerminateStateThread;
// function to execute user states in a worker thread
std::function<void(void)> fWork;

19
fairmq/FairMQTransportFactory.h
View File

@ -27,22 +27,39 @@
#include "FairMQTransports.h"
class FairMQChannel;
class FairMQProgOptions;
class FairMQTransportFactory
{
public:
/// Initialize transport
virtual void Initialize(const FairMQProgOptions* config) = 0;
/// Create an empty message (e.g. for receiving)
virtual FairMQMessagePtr CreateMessage() const = 0;
/// Create an empty of a specified size
virtual FairMQMessagePtr CreateMessage(const size_t size) const = 0;
/// Create a message from a supplied buffer, size and a deallocation callback
virtual FairMQMessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) const = 0;
virtual FairMQSocketPtr CreateSocket(const std::string& type, const std::string& name, const int numIoThreads, const std::string& id = "") const = 0;
/// Create a socket
virtual FairMQSocketPtr CreateSocket(const std::string& type, const std::string& name, const std::string& id = "") const = 0;
/// Create a poller for all device channels
virtual FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const = 0;
/// Create a poller for all device channels
virtual FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const = 0;
/// Create a poller for two sockets
virtual FairMQPollerPtr CreatePoller(const FairMQSocket& cmdSocket, const FairMQSocket& dataSocket) const = 0;
/// Get transport type
virtual FairMQ::Transport GetType() const = 0;
/// Shutdown transport (stop transfers, get ready for complete shutdown)
virtual void Shutdown() = 0;
/// Terminate transport (complete shutdown)
virtual void Terminate() = 0;
virtual ~FairMQTransportFactory() {};
};

12
fairmq/nanomsg/FairMQSocketNN.cxx
View File

@ -31,7 +31,7 @@ using namespace std;
atomic<bool> FairMQSocketNN::fInterrupted(false);
FairMQSocketNN::FairMQSocketNN(const string& type, const string& name, const int numIoThreads, const string& id /*= ""*/)
FairMQSocketNN::FairMQSocketNN(const string& type, const string& name, const string& id /*= ""*/)
: FairMQSocket(0, 0, NN_DONTWAIT)
, fSocket(-1)
, fId()
@ -42,11 +42,6 @@ FairMQSocketNN::FairMQSocketNN(const string& type, const string& name, const int
{
fId = id + "." + name + "." + type;
if (numIoThreads > 1)
{
LOG(INFO) << "number of I/O threads is not used in nanomsg";
}
if (type == "router" || type == "dealer")
{
// Additional info about using the sockets ROUTER and DEALER with nanomsg can be found in:
@ -370,11 +365,6 @@ void FairMQSocketNN::Close()
nn_close(fSocket);
}
void FairMQSocketNN::Terminate()
{
nn_term();
}
void FairMQSocketNN::Interrupt()
{
fInterrupted = true;

3
fairmq/nanomsg/FairMQSocketNN.h
View File

@ -24,7 +24,7 @@
class FairMQSocketNN : public FairMQSocket
{
public:
FairMQSocketNN(const std::string& type, const std::string& name, const int numIoThreads, const std::string& id = ""); // numIoThreads is not used in nanomsg.
FairMQSocketNN(const std::string& type, const std::string& name, const std::string& id = "");
FairMQSocketNN(const FairMQSocketNN&) = delete;
FairMQSocketNN operator=(const FairMQSocketNN&) = delete;
@ -42,7 +42,6 @@ class FairMQSocketNN : public FairMQSocket
virtual void* GetSocket() const;
virtual int GetSocket(int nothing) const;
virtual void Close();
virtual void Terminate();
virtual void Interrupt();
virtual void Resume();

22
fairmq/nanomsg/FairMQTransportFactoryNN.cxx
View File

@ -13,6 +13,9 @@
*/
#include "FairMQTransportFactoryNN.h"
#include "../options/FairMQProgOptions.h"
#include <nanomsg/nn.h>
using namespace std;
@ -23,6 +26,11 @@ FairMQTransportFactoryNN::FairMQTransportFactoryNN()
LOG(DEBUG) << "Transport: Using nanomsg library";
}
void FairMQTransportFactoryNN::Initialize(const FairMQProgOptions* config)
{
// nothing to do for nanomsg, transport is ready to be used any time (until nn_term()).
}
FairMQMessagePtr FairMQTransportFactoryNN::CreateMessage() const
{
return unique_ptr<FairMQMessage>(new FairMQMessageNN());
@ -38,9 +46,9 @@ FairMQMessagePtr FairMQTransportFactoryNN::CreateMessage(void* data, const size_
return unique_ptr<FairMQMessage>(new FairMQMessageNN(data, size, ffn, hint));
}
FairMQSocketPtr FairMQTransportFactoryNN::CreateSocket(const string& type, const string& name, const int numIoThreads, const string& id /*= ""*/) const
FairMQSocketPtr FairMQTransportFactoryNN::CreateSocket(const string& type, const string& name, const string& id /*= ""*/) const
{
return unique_ptr<FairMQSocket>(new FairMQSocketNN(type, name, numIoThreads, id));
return unique_ptr<FairMQSocket>(new FairMQSocketNN(type, name, id));
}
FairMQPollerPtr FairMQTransportFactoryNN::CreatePoller(const vector<FairMQChannel>& channels) const
@ -58,6 +66,16 @@ FairMQPollerPtr FairMQTransportFactoryNN::CreatePoller(const FairMQSocket& cmdSo
return unique_ptr<FairMQPoller>(new FairMQPollerNN(cmdSocket, dataSocket));
}
void FairMQTransportFactoryNN::Shutdown()
{
// nothing to do for nanomsg, transport is ready to be terminated any time.
}
void FairMQTransportFactoryNN::Terminate()
{
nn_term();
}
FairMQ::Transport FairMQTransportFactoryNN::GetType() const
{
return fTransportType;

9
fairmq/nanomsg/FairMQTransportFactoryNN.h
View File

@ -28,11 +28,13 @@ class FairMQTransportFactoryNN : public FairMQTransportFactory
public:
FairMQTransportFactoryNN();
virtual void Initialize(const FairMQProgOptions* config);
virtual FairMQMessagePtr CreateMessage() const;
virtual FairMQMessagePtr CreateMessage(const size_t size) const;
virtual FairMQMessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = NULL) const;
virtual FairMQMessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) const;
virtual FairMQSocketPtr CreateSocket(const std::string& type, const std::string& name, const int numIoThreads, const std::string& id = "") const;
virtual FairMQSocketPtr CreateSocket(const std::string& type, const std::string& name, const std::string& id = "") const;
virtual FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const;
virtual FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const;
@ -40,6 +42,9 @@ class FairMQTransportFactoryNN : public FairMQTransportFactory
virtual FairMQ::Transport GetType() const;
virtual void Shutdown();
virtual void Terminate();
virtual ~FairMQTransportFactoryNN() {};
private:

5
fairmq/runFairMQDevice.h
View File

@ -52,6 +52,11 @@ int main(int argc, char** argv)
config.ParseAll(argc, argv);
std::unique_ptr<FairMQDevice> device(getDevice(config));
if (!device)
{
LOG(ERROR) << "getDevice(): no valid device provided. Exiting.";
return 1;
}
int result = runStateMachine(*device, config);
if (result > 0)

83
fairmq/shmem/FairMQContextSHM.cxx
View File

@ -1,83 +0,0 @@
/********************************************************************************
* Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH *
* *
* This software is distributed under the terms of the *
* GNU Lesser General Public Licence version 3 (LGPL) version 3, *
* copied verbatim in the file "LICENSE" *
********************************************************************************/
#include <zmq.h>
#include <cstdio>
#include <boost/interprocess/managed_shared_memory.hpp>
#include "FairMQLogger.h"
#include "FairMQContextSHM.h"
#include "FairMQShmManager.h"
using namespace FairMQ::shmem;
FairMQContextSHM::FairMQContextSHM(int numIoThreads)
: fContext()
{
fContext = zmq_ctx_new();
if (fContext == NULL)
{
LOG(ERROR) << "failed creating context, reason: " << zmq_strerror(errno);
exit(EXIT_FAILURE);
}
if (zmq_ctx_set(fContext, ZMQ_IO_THREADS, numIoThreads) != 0)
{
LOG(ERROR) << "failed configuring context, reason: " << zmq_strerror(errno);
}
// Set the maximum number of allowed sockets on the context.
if (zmq_ctx_set(fContext, ZMQ_MAX_SOCKETS, 10000) != 0)
{
LOG(ERROR) << "failed configuring context, reason: " << zmq_strerror(errno);
}
Manager::Instance().InitializeSegment("open_or_create", "FairMQSharedMemory", 2000000000);
LOG(INFO) << "Created/Opened shared memory segment of 2,000,000,000 bytes. Available are " << Manager::Instance().Segment()->get_free_memory() << " bytes.";
}
FairMQContextSHM::~FairMQContextSHM()
{
Close();
if (boost::interprocess::shared_memory_object::remove("FairMQSharedMemory"))
{
fprintf(stderr, "Successfully removed shared memory after the device has stopped.\n");
}
else
{
fprintf(stderr, "Did not remove shared memory after the device stopped. Already removed?\n");
}
}
void* FairMQContextSHM::GetContext()
{
return fContext;
}
void FairMQContextSHM::Close()
{
if (fContext == NULL)
{
return;
}
if (zmq_ctx_destroy(fContext) != 0)
{
if (errno == EINTR)
{
LOG(ERROR) << " failed closing context, reason: " << zmq_strerror(errno);
}
else
{
fContext = NULL;
return;
}
}
}

27
fairmq/shmem/FairMQContextSHM.h
View File

@ -1,27 +0,0 @@
/********************************************************************************
* Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH *
* *
* This software is distributed under the terms of the *
* GNU Lesser General Public Licence version 3 (LGPL) version 3, *
* copied verbatim in the file "LICENSE" *
********************************************************************************/
#ifndef FAIRMQCONTEXTSHM_H_
#define FAIRMQCONTEXTSHM_H_
class FairMQContextSHM
{
public:
/// Constructor
FairMQContextSHM(int numIoThreads);
FairMQContextSHM(const FairMQContextSHM&) = delete;
FairMQContextSHM operator=(const FairMQContextSHM&) = delete;
virtual ~FairMQContextSHM();
void* GetContext();
void Close();
private:
void* fContext;
};
#endif /* FAIRMQCONTEXTSHM_H_ */

27
fairmq/shmem/FairMQSocketSHM.cxx
View File

@ -16,12 +16,9 @@
using namespace std;
using namespace FairMQ::shmem;
// Context to hold the ZeroMQ sockets
unique_ptr<FairMQContextSHM> FairMQSocketSHM::fContext; // = unique_ptr<FairMQContextSHM>(new FairMQContextSHM(1));
bool FairMQSocketSHM::fContextInitialized = false;
atomic<bool> FairMQSocketSHM::fInterrupted(false);
FairMQSocketSHM::FairMQSocketSHM(const string& type, const string& name, const int numIoThreads, const string& id /*= ""*/)
FairMQSocketSHM::FairMQSocketSHM(const string& type, const string& name, const string& id /*= ""*/, void* context)
: FairMQSocket(ZMQ_SNDMORE, ZMQ_RCVMORE, ZMQ_DONTWAIT)
, fSocket(NULL)
, fId()
@ -32,18 +29,8 @@ FairMQSocketSHM::FairMQSocketSHM(const string& type, const string& name, const i
{
fId = id + "." + name + "." + type;
if (!fContextInitialized)
{
fContext = unique_ptr<FairMQContextSHM>(new FairMQContextSHM(1));
fContextInitialized = true;
}
if (zmq_ctx_set(fContext->GetContext(), ZMQ_IO_THREADS, numIoThreads) != 0)
{
LOG(ERROR) << "Failed configuring context, reason: " << zmq_strerror(errno);
}
fSocket = zmq_socket(fContext->GetContext(), GetConstant(type));
assert(context);
fSocket = zmq_socket(context, GetConstant(type));
if (fSocket == NULL)
{
@ -409,14 +396,6 @@ void FairMQSocketSHM::Close()
fSocket = NULL;
}
void FairMQSocketSHM::Terminate()
{
if (zmq_ctx_destroy(fContext->GetContext()) != 0)
{
LOG(ERROR) << "Failed terminating context, reason: " << zmq_strerror(errno);
}
}
void FairMQSocketSHM::Interrupt()
{
FairMQMessageSHM::fInterrupted = true;

6
fairmq/shmem/FairMQSocketSHM.h
View File

@ -14,13 +14,12 @@
#include "FairMQSocket.h"
#include "FairMQMessage.h"
#include "FairMQContextSHM.h"
#include "FairMQShmManager.h"
class FairMQSocketSHM : public FairMQSocket
{
public:
FairMQSocketSHM(const std::string& type, const std::string& name, const int numIoThreads, const std::string& id = "");
FairMQSocketSHM(const std::string& type, const std::string& name, const std::string& id = "", void* context = nullptr);
FairMQSocketSHM(const FairMQSocketSHM&) = delete;
FairMQSocketSHM operator=(const FairMQSocketSHM&) = delete;
@ -38,7 +37,6 @@ class FairMQSocketSHM : public FairMQSocket
virtual void* GetSocket() const;
virtual int GetSocket(int nothing) const;
virtual void Close();
virtual void Terminate();
virtual void Interrupt();
virtual void Resume();
@ -68,8 +66,6 @@ class FairMQSocketSHM : public FairMQSocket
std::atomic<unsigned long> fMessagesTx;
std::atomic<unsigned long> fMessagesRx;
static std::unique_ptr<FairMQContextSHM> fContext;
static bool fContextInitialized;
static std::atomic<bool> fInterrupted;
};

85
fairmq/shmem/FairMQTransportFactorySHM.cxx
View File

@ -7,19 +7,59 @@
********************************************************************************/
#include "zmq.h"
#include <boost/version.hpp>
#include <boost/interprocess/managed_shared_memory.hpp>
#include "FairMQLogger.h"
#include "FairMQShmManager.h"
#include "FairMQTransportFactorySHM.h"
#include "../options/FairMQProgOptions.h"
using namespace std;
using namespace FairMQ::shmem;
FairMQ::Transport FairMQTransportFactorySHM::fTransportType = FairMQ::Transport::SHM;
FairMQTransportFactorySHM::FairMQTransportFactorySHM()
: fContext(nullptr)
{
int major, minor, patch;
zmq_version(&major, &minor, &patch);
LOG(DEBUG) << "Transport: Using ZeroMQ (" << major << "." << minor << "." << patch << ") & "
<< "boost::interprocess (" << (BOOST_VERSION / 100000) << "." << (BOOST_VERSION / 100 % 1000) << "." << (BOOST_VERSION % 100) << ")";
fContext = zmq_ctx_new();
if (!fContext)
{
LOG(ERROR) << "failed creating context, reason: " << zmq_strerror(errno);
exit(EXIT_FAILURE);
}
}
void FairMQTransportFactorySHM::Initialize(const FairMQProgOptions* config)
{
int numIoThreads = 1;
if (config)
{
numIoThreads = config->GetValue<int>("io-threads");
}
else
{
LOG(WARN) << "shmem: FairMQProgOptions not available! Using defaults.";
}
if (zmq_ctx_set(fContext, ZMQ_IO_THREADS, numIoThreads) != 0)
{
LOG(ERROR) << "failed configuring context, reason: " << zmq_strerror(errno);
}
// Set the maximum number of allowed sockets on the context.
if (zmq_ctx_set(fContext, ZMQ_MAX_SOCKETS, 10000) != 0)
{
LOG(ERROR) << "failed configuring context, reason: " << zmq_strerror(errno);
}
Manager::Instance().InitializeSegment("open_or_create", "FairMQSharedMemory", 2000000000);
LOG(DEBUG) << "shmem: created/opened shared memory segment of 2000000000 bytes. Available are " << Manager::Instance().Segment()->get_free_memory() << " bytes.";
}
FairMQMessagePtr FairMQTransportFactorySHM::CreateMessage() const
@ -37,9 +77,10 @@ FairMQMessagePtr FairMQTransportFactorySHM::CreateMessage(void* data, const size
return unique_ptr<FairMQMessage>(new FairMQMessageSHM(data, size, ffn, hint));
}
FairMQSocketPtr FairMQTransportFactorySHM::CreateSocket(const string& type, const string& name, const int numIoThreads, const string& id /*= ""*/) const
FairMQSocketPtr FairMQTransportFactorySHM::CreateSocket(const string& type, const string& name, const string& id /*= ""*/) const
{
return unique_ptr<FairMQSocket>(new FairMQSocketSHM(type, name, numIoThreads, id));
assert(fContext);
return unique_ptr<FairMQSocket>(new FairMQSocketSHM(type, name, id, fContext));
}
FairMQPollerPtr FairMQTransportFactorySHM::CreatePoller(const vector<FairMQChannel>& channels) const
@ -57,6 +98,46 @@ FairMQPollerPtr FairMQTransportFactorySHM::CreatePoller(const FairMQSocket& cmdS
return unique_ptr<FairMQPoller>(new FairMQPollerSHM(cmdSocket, dataSocket));
}
void FairMQTransportFactorySHM::Shutdown()
{
if (zmq_ctx_shutdown(fContext) != 0)
{
LOG(ERROR) << "shmem: failed shutting down context, reason: " << zmq_strerror(errno);
}
}
void FairMQTransportFactorySHM::Terminate()
{
if (fContext)
{
if (zmq_ctx_term(fContext) != 0)
{
if (errno == EINTR)
{
LOG(ERROR) << "shmem: failed closing context, reason: " << zmq_strerror(errno);
}
else
{
fContext = NULL;
return;
}
}
}
else
{
LOG(ERROR) << "shmem: Terminate(): context now available for shutdown";
}
if (boost::interprocess::shared_memory_object::remove("FairMQSharedMemory"))
{
LOG(DEBUG) << "shmem: successfully removed shared memory segment after the device has stopped.";
}
else
{
LOG(DEBUG) << "shmem: did not remove shared memory segment after the device stopped. Already removed?";
}
}
FairMQ::Transport FairMQTransportFactorySHM::GetType() const
{
return fTransportType;

11
fairmq/shmem/FairMQTransportFactorySHM.h
View File

@ -12,7 +12,6 @@
#include <string>
#include "FairMQTransportFactory.h"
#include "FairMQContextSHM.h"
#include "FairMQMessageSHM.h"
#include "FairMQSocketSHM.h"
#include "FairMQPollerSHM.h"
@ -22,11 +21,13 @@ class FairMQTransportFactorySHM : public FairMQTransportFactory
public:
FairMQTransportFactorySHM();
virtual void Initialize(const FairMQProgOptions* config);
virtual FairMQMessagePtr CreateMessage() const;
virtual FairMQMessagePtr CreateMessage(const size_t size) const;
virtual FairMQMessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = NULL) const;
virtual FairMQMessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) const;
virtual FairMQSocketPtr CreateSocket(const std::string& type, const std::string& name, const int numIoThreads, const std::string& id = "") const;
virtual FairMQSocketPtr CreateSocket(const std::string& type, const std::string& name, const std::string& id = "") const;
virtual FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const;
virtual FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const;
@ -34,10 +35,14 @@ class FairMQTransportFactorySHM : public FairMQTransportFactory
virtual FairMQ::Transport GetType() const;
virtual void Shutdown();
virtual void Terminate();
virtual ~FairMQTransportFactorySHM() {};
private:
static FairMQ::Transport fTransportType;
void* fContext;
};
#endif /* FAIRMQTRANSPORTFACTORYSHM_H_ */

71
fairmq/zeromq/FairMQContextZMQ.cxx
View File

@ -1,71 +0,0 @@
/********************************************************************************
* Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH *
* *
* This software is distributed under the terms of the *
* GNU Lesser General Public Licence version 3 (LGPL) version 3, *
* copied verbatim in the file "LICENSE" *
********************************************************************************/
/**
* FairMQContextZMQ.cxx
*
* @since 2012-12-05
* @author D. Klein, A. Rybalchenko
*/
#include <zmq.h>
#include "FairMQLogger.h"
#include "FairMQContextZMQ.h"
FairMQContextZMQ::FairMQContextZMQ(int numIoThreads)
: fContext()
{
fContext = zmq_ctx_new();
if (fContext == NULL)
{
LOG(ERROR) << "failed creating context, reason: " << zmq_strerror(errno);
exit(EXIT_FAILURE);
}
if (zmq_ctx_set(fContext, ZMQ_IO_THREADS, numIoThreads) != 0)
{
LOG(ERROR) << "failed configuring context, reason: " << zmq_strerror(errno);
}
// Set the maximum number of allowed sockets on the context.
if (zmq_ctx_set(fContext, ZMQ_MAX_SOCKETS, 10000) != 0)
{
LOG(ERROR) << "failed configuring context, reason: " << zmq_strerror(errno);
}
}
FairMQContextZMQ::~FairMQContextZMQ()
{
Close();
}
void* FairMQContextZMQ::GetContext()
{
return fContext;
}
void FairMQContextZMQ::Close()
{
if (fContext == NULL)
{
return;
}
if (zmq_ctx_destroy(fContext) != 0)
{
if (errno == EINTR)
{
LOG(ERROR) << " failed closing context, reason: " << zmq_strerror(errno);
}
else
{
fContext = NULL;
return;
}
}
}

34
fairmq/zeromq/FairMQContextZMQ.h
View File

@ -1,34 +0,0 @@
/********************************************************************************
* Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH *
* *
* This software is distributed under the terms of the *
* GNU Lesser General Public Licence version 3 (LGPL) version 3, *
* copied verbatim in the file "LICENSE" *
********************************************************************************/
/**
* FairMQContextZMQ.h
*
* @since 2012-12-05
* @author D. Klein, A. Rybalchenko
*/
#ifndef FAIRMQCONTEXTZMQ_H_
#define FAIRMQCONTEXTZMQ_H_
class FairMQContextZMQ
{
public:
/// Constructor
FairMQContextZMQ(int numIoThreads);
FairMQContextZMQ(const FairMQContextZMQ&) = delete;
FairMQContextZMQ operator=(const FairMQContextZMQ&) = delete;
virtual ~FairMQContextZMQ();
void* GetContext();
void Close();
private:
void* fContext;
};
#endif /* FAIRMQCONTEXTZMQ_H_ */

20
fairmq/zeromq/FairMQSocketZMQ.cxx
View File

@ -22,11 +22,9 @@
using namespace std;
// Context to hold the ZeroMQ sockets
unique_ptr<FairMQContextZMQ> FairMQSocketZMQ::fContext = unique_ptr<FairMQContextZMQ>(new FairMQContextZMQ(1));
atomic<bool> FairMQSocketZMQ::fInterrupted(false);
FairMQSocketZMQ::FairMQSocketZMQ(const string& type, const string& name, const int numIoThreads, const string& id /*= ""*/)
FairMQSocketZMQ::FairMQSocketZMQ(const string& type, const string& name, const string& id /*= ""*/, void* context)
: FairMQSocket(ZMQ_SNDMORE, ZMQ_RCVMORE, ZMQ_DONTWAIT)
, fSocket(NULL)
, fId()
@ -37,12 +35,8 @@ FairMQSocketZMQ::FairMQSocketZMQ(const string& type, const string& name, const i
{
fId = id + "." + name + "." + type;
if (zmq_ctx_set(fContext->GetContext(), ZMQ_IO_THREADS, numIoThreads) != 0)
{
LOG(ERROR) << "Failed configuring context, reason: " << zmq_strerror(errno);
}
fSocket = zmq_socket(fContext->GetContext(), GetConstant(type));
assert(context);
fSocket = zmq_socket(context, GetConstant(type));
if (fSocket == NULL)
{
@ -346,14 +340,6 @@ void FairMQSocketZMQ::Close()
fSocket = NULL;
}
void FairMQSocketZMQ::Terminate()
{
if (zmq_ctx_destroy(fContext->GetContext()) != 0)
{
LOG(ERROR) << "Failed terminating context, reason: " << zmq_strerror(errno);
}
}
void FairMQSocketZMQ::Interrupt()
{
fInterrupted = true;

5
fairmq/zeromq/FairMQSocketZMQ.h
View File

@ -21,12 +21,11 @@
#include "FairMQSocket.h"
#include "FairMQMessage.h"
#include "FairMQContextZMQ.h"
class FairMQSocketZMQ : public FairMQSocket
{
public:
FairMQSocketZMQ(const std::string& type, const std::string& name, const int numIoThreads, const std::string& id = "");
FairMQSocketZMQ(const std::string& type, const std::string& name, const std::string& id = "", void* context = nullptr);
FairMQSocketZMQ(const FairMQSocketZMQ&) = delete;
FairMQSocketZMQ operator=(const FairMQSocketZMQ&) = delete;
@ -44,7 +43,6 @@ class FairMQSocketZMQ : public FairMQSocket
virtual void* GetSocket() const;
virtual int GetSocket(int nothing) const;
virtual void Close();
virtual void Terminate();
virtual void Interrupt();
virtual void Resume();
@ -74,7 +72,6 @@ class FairMQSocketZMQ : public FairMQSocket
std::atomic<unsigned long> fMessagesTx;
std::atomic<unsigned long> fMessagesRx;
static std::unique_ptr<FairMQContextZMQ> fContext;
static std::atomic<bool> fInterrupted;
};

68
fairmq/zeromq/FairMQTransportFactoryZMQ.cxx
View File

@ -15,16 +15,48 @@
#include "zmq.h"
#include "FairMQTransportFactoryZMQ.h"
#include "../options/FairMQProgOptions.h"
using namespace std;
FairMQ::Transport FairMQTransportFactoryZMQ::fTransportType = FairMQ::Transport::ZMQ;
FairMQTransportFactoryZMQ::FairMQTransportFactoryZMQ()
: fContext(zmq_ctx_new())
{
int major, minor, patch;
zmq_version(&major, &minor, &patch);
LOG(DEBUG) << "Transport: Using ZeroMQ library, version: " << major << "." << minor << "." << patch;
if (!fContext)
{
LOG(ERROR) << "failed creating context, reason: " << zmq_strerror(errno);
exit(EXIT_FAILURE);
}
}
void FairMQTransportFactoryZMQ::Initialize(const FairMQProgOptions* config)
{
int numIoThreads = 1;
if (config)
{
numIoThreads = config->GetValue<int>("io-threads");
}
else
{
LOG(WARN) << "zeromq: FairMQProgOptions not available! Using defaults.";
}
if (zmq_ctx_set(fContext, ZMQ_IO_THREADS, numIoThreads) != 0)
{
LOG(ERROR) << "failed configuring context, reason: " << zmq_strerror(errno);
}
// Set the maximum number of allowed sockets on the context.
if (zmq_ctx_set(fContext, ZMQ_MAX_SOCKETS, 10000) != 0)
{
LOG(ERROR) << "failed configuring context, reason: " << zmq_strerror(errno);
}
}
FairMQMessagePtr FairMQTransportFactoryZMQ::CreateMessage() const
@ -42,9 +74,10 @@ FairMQMessagePtr FairMQTransportFactoryZMQ::CreateMessage(void* data, const size
return unique_ptr<FairMQMessage>(new FairMQMessageZMQ(data, size, ffn, hint));
}
FairMQSocketPtr FairMQTransportFactoryZMQ::CreateSocket(const string& type, const string& name, const int numIoThreads, const string& id /*= ""*/) const
FairMQSocketPtr FairMQTransportFactoryZMQ::CreateSocket(const string& type, const string& name, const string& id /*= ""*/) const
{
return unique_ptr<FairMQSocket>(new FairMQSocketZMQ(type, name, numIoThreads, id));
assert(fContext);
return unique_ptr<FairMQSocket>(new FairMQSocketZMQ(type, name, id, fContext));
}
FairMQPollerPtr FairMQTransportFactoryZMQ::CreatePoller(const vector<FairMQChannel>& channels) const
@ -66,3 +99,34 @@ FairMQ::Transport FairMQTransportFactoryZMQ::GetType() const
{
return fTransportType;
}
void FairMQTransportFactoryZMQ::Shutdown()
{
if (zmq_ctx_shutdown(fContext) != 0)
{
LOG(ERROR) << "zeromq: failed shutting down context, reason: " << zmq_strerror(errno);
}
}
void FairMQTransportFactoryZMQ::Terminate()
{
if (fContext)
{
if (zmq_ctx_term(fContext) != 0)
{
if (errno == EINTR)
{
LOG(ERROR) << " failed closing context, reason: " << zmq_strerror(errno);
}
else
{
fContext = nullptr;
return;
}
}
}
else
{
LOG(ERROR) << "shmem: Terminate(): context now available for shutdown";
}
}

11
fairmq/zeromq/FairMQTransportFactoryZMQ.h
View File

@ -19,7 +19,6 @@
#include <string>
#include "FairMQTransportFactory.h"
#include "FairMQContextZMQ.h"
#include "FairMQMessageZMQ.h"
#include "FairMQSocketZMQ.h"
#include "FairMQPollerZMQ.h"
@ -29,11 +28,13 @@ class FairMQTransportFactoryZMQ : public FairMQTransportFactory
public:
FairMQTransportFactoryZMQ();
virtual void Initialize(const FairMQProgOptions* config);
virtual FairMQMessagePtr CreateMessage() const;
virtual FairMQMessagePtr CreateMessage(const size_t size) const;
virtual FairMQMessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = NULL) const;
virtual FairMQMessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) const;
virtual FairMQSocketPtr CreateSocket(const std::string& type, const std::string& name, const int numIoThreads, const std::string& id = "") const;
virtual FairMQSocketPtr CreateSocket(const std::string& type, const std::string& name, const std::string& id = "") const;
virtual FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const;
virtual FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const;
@ -41,10 +42,14 @@ class FairMQTransportFactoryZMQ : public FairMQTransportFactory
virtual FairMQ::Transport GetType() const;
virtual void Shutdown();
virtual void Terminate();
virtual ~FairMQTransportFactoryZMQ() {};
private:
static FairMQ::Transport fTransportType;
void* fContext;
};
#endif /* FAIRMQTRANSPORTFACTORYZMQ_H_ */