Update multi-part features (nanomsg) and various fixes

- Implement nanomsg multipart with MessagePack.
 - Use the MessagePack from FairSoft and handle not found case.
 - Update splitter, merger and proxy devices to handle multi-part.
 - Let FairMQParts.At() return pointer reference (can be used for moving).
 - Add missing const specifier in the message interface.
 - Add transmit kernel size setting to channels (ZMQ_SNDBUF).
 - Remove FairMQBuffer device.
 - Remove old multi-part methods from Tutorial3 example (to be replaced with Parts API).
 - Make callback mandatory for newMsg(data, size, callback).
 - Add missing <vector> include in FairMQSocket.
This commit is contained in:
Alexey Rybalchenko
2016-03-21 09:59:00 +01:00
parent 4ca66e33da
commit 732373faa2
25 changed files with 436 additions and 381 deletions

View File

@@ -17,6 +17,7 @@
#include <zmq.h>
#include "FairMQSocketZMQ.h"
#include "FairMQMessageZMQ.h"
#include "FairMQLogger.h"
using namespace std;
@@ -24,7 +25,7 @@ using namespace std;
// Context to hold the ZeroMQ sockets
boost::shared_ptr<FairMQContextZMQ> FairMQSocketZMQ::fContext = boost::shared_ptr<FairMQContextZMQ>(new FairMQContextZMQ(1));
FairMQSocketZMQ::FairMQSocketZMQ(const string& type, const string& name, const int numIoThreads, const std::string& id /*= ""*/)
FairMQSocketZMQ::FairMQSocketZMQ(const string& type, const string& name, const int numIoThreads, const string& id /*= ""*/)
: FairMQSocket(ZMQ_SNDMORE, ZMQ_RCVMORE, ZMQ_DONTWAIT)
, fSocket(NULL)
, fId()
@@ -68,6 +69,8 @@ FairMQSocketZMQ::FairMQSocketZMQ(const string& type, const string& name, const i
LOG(ERROR) << "Failed setting ZMQ_SUBSCRIBE socket option, reason: " << zmq_strerror(errno);
}
}
// LOG(INFO) << "created socket " << fId;
}
string FairMQSocketZMQ::GetId()
@@ -147,6 +150,52 @@ int FairMQSocketZMQ::Send(FairMQMessage* msg, const int flags)
return nbytes;
}
int64_t FairMQSocketZMQ::Send(const vector<unique_ptr<FairMQMessage>>& msgVec)
{
// Sending vector typicaly handles more then one part
if (msgVec.size() > 1)
{
int64_t totalSize = 0;
for (unsigned int i = 0; i < msgVec.size() - 1; ++i)
{
int nbytes = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec[i]->GetMessage()), fSocket, ZMQ_SNDMORE);
if (nbytes >= 0)
{
totalSize += nbytes;
fBytesTx += nbytes;
}
else
{
return nbytes;
}
}
int n = zmq_msg_send(static_cast<zmq_msg_t*>(msgVec.back()->GetMessage()), fSocket, 0);
if (n >= 0)
{
totalSize += n;
}
else
{
return n;
}
// store statistics on how many messages have been sent (handle all parts as a single message)
++fMessagesTx;
return totalSize;
} // If there's only one part, send it as a regular message
else if (msgVec.size() == 1)
{
return zmq_msg_send(static_cast<zmq_msg_t*>(msgVec.back()->GetMessage()), fSocket, 0);
}
else // if the vector is empty, something might be wrong
{
LOG(WARN) << "Will not send empty vector";
return -1;
}
}
int FairMQSocketZMQ::Receive(FairMQMessage* msg, const string& flag)
{
int nbytes = zmq_msg_recv(static_cast<zmq_msg_t*>(msg->GetMessage()), fSocket, GetConstant(flag));
@@ -191,6 +240,44 @@ int FairMQSocketZMQ::Receive(FairMQMessage* msg, const int flags)
return nbytes;
}
int64_t FairMQSocketZMQ::Receive(vector<unique_ptr<FairMQMessage>>& msgVec)
{
// Warn if the vector is filled before Receive() and empty it.
if (msgVec.size() > 0)
{
LOG(WARN) << "Message vector contains elements before Receive(), they will be deleted!";
msgVec.clear();
}
int64_t totalSize = 0;
int64_t more = 0;
do
{
unique_ptr<FairMQMessage> part(new FairMQMessageZMQ());
int nbytes = zmq_msg_recv(static_cast<zmq_msg_t*>(part->GetMessage()), fSocket, 0);
if (nbytes >= 0)
{
msgVec.push_back(move(part));
totalSize += nbytes;
fBytesRx += nbytes;
}
else
{
return nbytes;
}
size_t more_size = sizeof(more);
zmq_getsockopt(fSocket, ZMQ_RCVMORE, &more, &more_size);
}
while (more);
// store statistics on how many messages have been received (handle all parts as a single message)
++fMessagesRx;
return totalSize;
}
void FairMQSocketZMQ::Close()
{
// LOG(DEBUG) << "Closing socket " << fId;
@@ -414,6 +501,10 @@ int FairMQSocketZMQ::GetConstant(const string& constant)
return ZMQ_SNDHWM;
if (constant == "rcv-hwm")
return ZMQ_RCVHWM;
if (constant == "snd-size")
return ZMQ_SNDBUF;
if (constant == "rcv-size")
return ZMQ_RCVBUF;
if (constant == "snd-more")
return ZMQ_SNDMORE;
if (constant == "rcv-more")