mirror of
https://github.com/FairRootGroup/FairMQ.git
synced 2025-10-15 09:31:45 +00:00
Remove nanomsg transport
This commit is contained in:
@@ -197,16 +197,6 @@ if(BUILD_FAIRMQ)
|
||||
zeromq/FairMQTransportFactoryZMQ.h
|
||||
)
|
||||
|
||||
if(BUILD_NANOMSG_TRANSPORT)
|
||||
set(FAIRMQ_PRIVATE_HEADER_FILES ${FAIRMQ_PRIVATE_HEADER_FILES}
|
||||
nanomsg/FairMQMessageNN.h
|
||||
nanomsg/FairMQPollerNN.h
|
||||
nanomsg/FairMQUnmanagedRegionNN.h
|
||||
nanomsg/FairMQSocketNN.h
|
||||
nanomsg/FairMQTransportFactoryNN.h
|
||||
)
|
||||
endif()
|
||||
|
||||
if(BUILD_OFI_TRANSPORT)
|
||||
set(FAIRMQ_PRIVATE_HEADER_FILES ${FAIRMQ_PRIVATE_HEADER_FILES}
|
||||
ofi/Context.h
|
||||
@@ -257,16 +247,6 @@ if(BUILD_FAIRMQ)
|
||||
MemoryResources.cxx
|
||||
)
|
||||
|
||||
if(BUILD_NANOMSG_TRANSPORT)
|
||||
set(FAIRMQ_SOURCE_FILES ${FAIRMQ_SOURCE_FILES}
|
||||
nanomsg/FairMQMessageNN.cxx
|
||||
nanomsg/FairMQPollerNN.cxx
|
||||
nanomsg/FairMQUnmanagedRegionNN.cxx
|
||||
nanomsg/FairMQSocketNN.cxx
|
||||
nanomsg/FairMQTransportFactoryNN.cxx
|
||||
)
|
||||
endif()
|
||||
|
||||
if(BUILD_OFI_TRANSPORT)
|
||||
set(FAIRMQ_SOURCE_FILES ${FAIRMQ_SOURCE_FILES}
|
||||
ofi/Context.cxx
|
||||
@@ -307,9 +287,6 @@ if(BUILD_FAIRMQ)
|
||||
# preprocessor definitions #
|
||||
############################
|
||||
target_compile_definitions(${_target} PUBLIC BOOST_ERROR_CODE_HEADER_ONLY)
|
||||
if(BUILD_NANOMSG_TRANSPORT)
|
||||
target_compile_definitions(${_target} PRIVATE BUILD_NANOMSG_TRANSPORT)
|
||||
endif()
|
||||
if(BUILD_OFI_TRANSPORT)
|
||||
target_compile_definitions(${_target} PRIVATE BUILD_OFI_TRANSPORT)
|
||||
endif()
|
||||
@@ -330,16 +307,13 @@ if(BUILD_FAIRMQ)
|
||||
##################
|
||||
# link libraries #
|
||||
##################
|
||||
if(BUILD_NANOMSG_TRANSPORT)
|
||||
set(NANOMSG_DEPS nanomsg msgpackc-cxx)
|
||||
endif()
|
||||
if(BUILD_OFI_TRANSPORT)
|
||||
set(OFI_DEPS
|
||||
asiofi::asiofi
|
||||
Boost::container
|
||||
)
|
||||
endif()
|
||||
set(optional_deps ${NANOMSG_DEPS} ${OFI_DEPS})
|
||||
set(optional_deps ${OFI_DEPS})
|
||||
if(optional_deps)
|
||||
list(REMOVE_DUPLICATES optional_deps)
|
||||
endif()
|
||||
@@ -362,7 +336,6 @@ if(BUILD_FAIRMQ)
|
||||
|
||||
PRIVATE # only libFairMQ links against private dependencies
|
||||
libzmq
|
||||
${NANOMSG_DEPS}
|
||||
${OFI_DEPS}
|
||||
)
|
||||
set_target_properties(${_target} PROPERTIES
|
||||
|
@@ -127,8 +127,8 @@ class FairMQChannel
|
||||
/// @return Returns socket address (e.g. "tcp://127.0.0.1:5555" or "ipc://abc")
|
||||
std::string GetAddress() const;
|
||||
|
||||
/// Get channel transport name ("default", "zeromq", "nanomsg" or "shmem")
|
||||
/// @return Returns channel transport name (e.g. "default", "zeromq", "nanomsg" or "shmem")
|
||||
/// Get channel transport name ("default", "zeromq" or "shmem")
|
||||
/// @return Returns channel transport name (e.g. "default", "zeromq" or "shmem")
|
||||
std::string GetTransportName() const;
|
||||
|
||||
/// Get channel transport type
|
||||
@@ -184,7 +184,7 @@ class FairMQChannel
|
||||
void UpdateAddress(const std::string& address);
|
||||
|
||||
/// Set channel transport
|
||||
/// @param transport transport string ("default", "zeromq", "nanomsg" or "shmem")
|
||||
/// @param transport transport string ("default", "zeromq" or "shmem")
|
||||
void UpdateTransport(const std::string& transport);
|
||||
|
||||
/// Set socket send buffer size
|
||||
|
@@ -302,7 +302,7 @@ class FairMQDevice
|
||||
}
|
||||
|
||||
/// Adds a transport to the device if it doesn't exist
|
||||
/// @param transport Transport string ("zeromq"/"nanomsg"/"shmem")
|
||||
/// @param transport Transport string ("zeromq"/"shmem")
|
||||
std::shared_ptr<FairMQTransportFactory> AddTransport(const fair::mq::Transport transport);
|
||||
|
||||
/// Assigns config to the device
|
||||
@@ -417,7 +417,7 @@ class FairMQDevice
|
||||
int GetInitTimeoutInS() const { return fConfig->GetProperty<int>("init-timeout", DefaultInitTimeout); }
|
||||
|
||||
/// Sets the default transport for the device
|
||||
/// @param transport Transport string ("zeromq"/"nanomsg"/"shmem")
|
||||
/// @param transport Transport string ("zeromq"/"shmem")
|
||||
void SetTransport(const std::string& transport) { fConfig->SetProperty("transport", transport); }
|
||||
/// Gets the default transport name
|
||||
std::string GetTransportName() const { return fConfig->GetProperty<std::string>("transport", DefaultTransportName); }
|
||||
|
@@ -9,9 +9,6 @@
|
||||
#include <FairMQTransportFactory.h>
|
||||
#include <fairmq/shmem/TransportFactory.h>
|
||||
#include <zeromq/FairMQTransportFactoryZMQ.h>
|
||||
#ifdef BUILD_NANOMSG_TRANSPORT
|
||||
#include <nanomsg/FairMQTransportFactoryNN.h>
|
||||
#endif /* BUILD_NANOMSG_TRANSPORT */
|
||||
#ifdef BUILD_OFI_TRANSPORT
|
||||
#include <fairmq/ofi/TransportFactory.h>
|
||||
#endif
|
||||
@@ -43,11 +40,6 @@ auto FairMQTransportFactory::CreateTransportFactory(const string& type,
|
||||
} else if (type == "shmem") {
|
||||
return make_shared<fair::mq::shmem::TransportFactory>(finalId, config);
|
||||
}
|
||||
#ifdef BUILD_NANOMSG_TRANSPORT
|
||||
else if (type == "nanomsg") {
|
||||
return make_shared<FairMQTransportFactoryNN>(finalId, config);
|
||||
}
|
||||
#endif /* BUILD_NANOMSG_TRANSPORT */
|
||||
#ifdef BUILD_OFI_TRANSPORT
|
||||
else if (type == "ofi") {
|
||||
return make_shared<fair::mq::ofi::TransportFactory>(finalId, config);
|
||||
@@ -57,11 +49,8 @@ auto FairMQTransportFactory::CreateTransportFactory(const string& type,
|
||||
LOG(error) << "Unavailable transport requested: "
|
||||
<< "\"" << type << "\""
|
||||
<< ". Available are: "
|
||||
<< "\"zeromq\""
|
||||
<< "\"zeromq\","
|
||||
<< "\"shmem\""
|
||||
#ifdef BUILD_NANOMSG_TRANSPORT
|
||||
<< ", \"nanomsg\""
|
||||
#endif /* BUILD_NANOMSG_TRANSPORT */
|
||||
#ifdef BUILD_OFI_TRANSPORT
|
||||
<< ", and \"ofi\""
|
||||
#endif /* BUILD_OFI_TRANSPORT */
|
||||
|
@@ -24,7 +24,6 @@ enum class Transport
|
||||
{
|
||||
DEFAULT,
|
||||
ZMQ,
|
||||
NN,
|
||||
SHM,
|
||||
OFI
|
||||
};
|
||||
@@ -48,7 +47,6 @@ namespace mq
|
||||
static std::unordered_map<std::string, Transport> TransportTypes {
|
||||
{ "default", Transport::DEFAULT },
|
||||
{ "zeromq", Transport::ZMQ },
|
||||
{ "nanomsg", Transport::NN },
|
||||
{ "shmem", Transport::SHM },
|
||||
{ "ofi", Transport::OFI }
|
||||
};
|
||||
@@ -56,7 +54,6 @@ static std::unordered_map<std::string, Transport> TransportTypes {
|
||||
static std::unordered_map<Transport, std::string> TransportNames {
|
||||
{ Transport::DEFAULT, "default" },
|
||||
{ Transport::ZMQ, "zeromq" },
|
||||
{ Transport::NN, "nanomsg" },
|
||||
{ Transport::SHM, "shmem" },
|
||||
{ Transport::OFI, "ofi" }
|
||||
};
|
||||
|
@@ -1,227 +0,0 @@
|
||||
/********************************************************************************
|
||||
* Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH *
|
||||
* *
|
||||
* This software is distributed under the terms of the *
|
||||
* GNU Lesser General Public Licence (LGPL) version 3, *
|
||||
* copied verbatim in the file "LICENSE" *
|
||||
********************************************************************************/
|
||||
/**
|
||||
* FairMQMessageNN.cxx
|
||||
*
|
||||
* @since 2013-12-05
|
||||
* @author A. Rybalchenko
|
||||
*/
|
||||
|
||||
#include <cstring>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include <nanomsg/nn.h>
|
||||
|
||||
#include "FairMQMessageNN.h"
|
||||
#include "FairMQLogger.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
fair::mq::Transport FairMQMessageNN::fTransportType = fair::mq::Transport::NN;
|
||||
|
||||
FairMQMessageNN::FairMQMessageNN(FairMQTransportFactory* factory)
|
||||
: FairMQMessage{factory}
|
||||
, fMessage(nullptr)
|
||||
, fSize(0)
|
||||
, fHint(0)
|
||||
, fReceiving(false)
|
||||
, fRegionPtr(nullptr)
|
||||
{
|
||||
fMessage = nn_allocmsg(0, 0);
|
||||
if (!fMessage)
|
||||
{
|
||||
LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
|
||||
}
|
||||
}
|
||||
|
||||
FairMQMessageNN::FairMQMessageNN(const size_t size, FairMQTransportFactory* factory)
|
||||
: FairMQMessage{factory}
|
||||
, fMessage(nullptr)
|
||||
, fSize(0)
|
||||
, fHint(0)
|
||||
, fReceiving(false)
|
||||
, fRegionPtr(nullptr)
|
||||
{
|
||||
fMessage = nn_allocmsg(size, 0);
|
||||
if (!fMessage)
|
||||
{
|
||||
LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
|
||||
}
|
||||
fSize = size;
|
||||
}
|
||||
|
||||
|
||||
/* nanomsg does not offer support for creating a message out of an existing buffer,
|
||||
* therefore the following method is using memcpy. For more efficient handling,
|
||||
* create FairMQMessage object only with size parameter and fill it with data.
|
||||
* possible TODO: make this zero copy (will should then be as efficient as ZeroMQ).
|
||||
*/
|
||||
FairMQMessageNN::FairMQMessageNN(void* data, const size_t size, fairmq_free_fn* ffn, void* hint, FairMQTransportFactory* factory)
|
||||
: FairMQMessage{factory}
|
||||
, fMessage(nullptr)
|
||||
, fSize(0)
|
||||
, fHint(0)
|
||||
, fReceiving(false)
|
||||
, fRegionPtr(nullptr)
|
||||
{
|
||||
fMessage = nn_allocmsg(size, 0);
|
||||
if (!fMessage)
|
||||
{
|
||||
LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
|
||||
}
|
||||
else
|
||||
{
|
||||
memcpy(fMessage, data, size);
|
||||
fSize = size;
|
||||
if (ffn)
|
||||
{
|
||||
ffn(data, hint);
|
||||
}
|
||||
else
|
||||
{
|
||||
free(data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
FairMQMessageNN::FairMQMessageNN(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint, FairMQTransportFactory* factory)
|
||||
: FairMQMessage{factory}
|
||||
, fMessage(data)
|
||||
, fSize(size)
|
||||
, fHint(reinterpret_cast<size_t>(hint))
|
||||
, fReceiving(false)
|
||||
, fRegionPtr(region.get())
|
||||
{
|
||||
// currently nanomsg will copy the buffer (data) inside nn_sendmsg()
|
||||
}
|
||||
|
||||
void FairMQMessageNN::Rebuild()
|
||||
{
|
||||
CloseMessage();
|
||||
fReceiving = false;
|
||||
}
|
||||
|
||||
void FairMQMessageNN::Rebuild(const size_t size)
|
||||
{
|
||||
CloseMessage();
|
||||
fMessage = nn_allocmsg(size, 0);
|
||||
if (!fMessage)
|
||||
{
|
||||
LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
|
||||
}
|
||||
fSize = size;
|
||||
fReceiving = false;
|
||||
}
|
||||
|
||||
void FairMQMessageNN::Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
|
||||
{
|
||||
CloseMessage();
|
||||
fMessage = nn_allocmsg(size, 0);
|
||||
if (!fMessage)
|
||||
{
|
||||
LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
|
||||
}
|
||||
else
|
||||
{
|
||||
memcpy(fMessage, data, size);
|
||||
fSize = size;
|
||||
fReceiving = false;
|
||||
|
||||
if (ffn)
|
||||
{
|
||||
ffn(data, hint);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void* FairMQMessageNN::GetMessage() const
|
||||
{
|
||||
return fMessage;
|
||||
}
|
||||
|
||||
void* FairMQMessageNN::GetData() const
|
||||
{
|
||||
return fMessage;
|
||||
}
|
||||
|
||||
size_t FairMQMessageNN::GetSize() const
|
||||
{
|
||||
return fSize;
|
||||
}
|
||||
|
||||
bool FairMQMessageNN::SetUsedSize(const size_t size)
|
||||
{
|
||||
if (size <= fSize)
|
||||
{
|
||||
// with size smaller than original nanomsg will simply "chop" the data, avoiding reallocation
|
||||
fMessage = nn_reallocmsg(fMessage, size);
|
||||
fSize = size;
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG(error) << "cannot set used size higher than original.";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
void FairMQMessageNN::SetMessage(void* data, const size_t size)
|
||||
{
|
||||
fMessage = data;
|
||||
fSize = size;
|
||||
}
|
||||
|
||||
fair::mq::Transport FairMQMessageNN::GetType() const
|
||||
{
|
||||
return fTransportType;
|
||||
}
|
||||
|
||||
void FairMQMessageNN::Copy(const FairMQMessage& msg)
|
||||
{
|
||||
if (fMessage)
|
||||
{
|
||||
if (nn_freemsg(fMessage) < 0)
|
||||
{
|
||||
LOG(error) << "failed freeing message, reason: " << nn_strerror(errno);
|
||||
}
|
||||
}
|
||||
|
||||
size_t size = msg.GetSize();
|
||||
|
||||
fMessage = nn_allocmsg(size, 0);
|
||||
if (!fMessage)
|
||||
{
|
||||
LOG(error) << "failed allocating message, reason: " << nn_strerror(errno);
|
||||
}
|
||||
else
|
||||
{
|
||||
memcpy(fMessage, static_cast<const FairMQMessageNN&>(msg).GetMessage(), size);
|
||||
fSize = size;
|
||||
}
|
||||
}
|
||||
|
||||
void FairMQMessageNN::CloseMessage()
|
||||
{
|
||||
if (nn_freemsg(fMessage) < 0)
|
||||
{
|
||||
LOG(error) << "failed freeing message, reason: " << nn_strerror(errno);
|
||||
}
|
||||
else
|
||||
{
|
||||
fMessage = nullptr;
|
||||
fSize = 0;
|
||||
}
|
||||
}
|
||||
|
||||
FairMQMessageNN::~FairMQMessageNN()
|
||||
{
|
||||
if (fReceiving)
|
||||
{
|
||||
CloseMessage();
|
||||
}
|
||||
}
|
@@ -1,68 +0,0 @@
|
||||
/********************************************************************************
|
||||
* Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH *
|
||||
* *
|
||||
* This software is distributed under the terms of the *
|
||||
* GNU Lesser General Public Licence (LGPL) version 3, *
|
||||
* copied verbatim in the file "LICENSE" *
|
||||
********************************************************************************/
|
||||
/**
|
||||
* FairMQMessageNN.h
|
||||
*
|
||||
* @since 2013-12-05
|
||||
* @author A. Rybalchenko
|
||||
*/
|
||||
|
||||
#ifndef FAIRMQMESSAGENN_H_
|
||||
#define FAIRMQMESSAGENN_H_
|
||||
|
||||
#include <cstddef>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
|
||||
#include "FairMQMessage.h"
|
||||
#include "FairMQUnmanagedRegion.h"
|
||||
|
||||
class FairMQSocketNN;
|
||||
|
||||
class FairMQMessageNN final : public FairMQMessage
|
||||
{
|
||||
friend class FairMQSocketNN;
|
||||
|
||||
public:
|
||||
FairMQMessageNN(FairMQTransportFactory* factory = nullptr);
|
||||
FairMQMessageNN(const size_t size, FairMQTransportFactory* factory = nullptr);
|
||||
FairMQMessageNN(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr, FairMQTransportFactory* factory = nullptr);
|
||||
FairMQMessageNN(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0, FairMQTransportFactory* factory = nullptr);
|
||||
|
||||
FairMQMessageNN(const FairMQMessageNN&) = delete;
|
||||
FairMQMessageNN operator=(const FairMQMessageNN&) = delete;
|
||||
|
||||
void Rebuild() override;
|
||||
void Rebuild(const size_t size) override;
|
||||
void Rebuild(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override;
|
||||
|
||||
void* GetData() const override;
|
||||
size_t GetSize() const override;
|
||||
|
||||
bool SetUsedSize(const size_t size) override;
|
||||
|
||||
fair::mq::Transport GetType() const override;
|
||||
|
||||
void Copy(const FairMQMessage& msg) override;
|
||||
|
||||
~FairMQMessageNN() override;
|
||||
|
||||
private:
|
||||
void* fMessage;
|
||||
size_t fSize;
|
||||
size_t fHint;
|
||||
bool fReceiving;
|
||||
FairMQUnmanagedRegion* fRegionPtr;
|
||||
static fair::mq::Transport fTransportType;
|
||||
|
||||
void* GetMessage() const;
|
||||
void CloseMessage();
|
||||
void SetMessage(void* data, const size_t size);
|
||||
};
|
||||
|
||||
#endif /* FAIRMQMESSAGENN_H_ */
|
@@ -1,207 +0,0 @@
|
||||
/********************************************************************************
|
||||
* Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH *
|
||||
* *
|
||||
* This software is distributed under the terms of the *
|
||||
* GNU Lesser General Public Licence (LGPL) version 3, *
|
||||
* copied verbatim in the file "LICENSE" *
|
||||
********************************************************************************/
|
||||
/**
|
||||
* FairMQPollerNN.cxx
|
||||
*
|
||||
* @since 2014-01-23
|
||||
* @author A. Rybalchenko
|
||||
*/
|
||||
|
||||
#include <nanomsg/nn.h>
|
||||
#include <nanomsg/pipeline.h>
|
||||
#include <nanomsg/pubsub.h>
|
||||
#include <nanomsg/reqrep.h>
|
||||
#include <nanomsg/pair.h>
|
||||
|
||||
#include "FairMQPollerNN.h"
|
||||
#include "FairMQSocketNN.h"
|
||||
#include "FairMQLogger.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
FairMQPollerNN::FairMQPollerNN(const vector<FairMQChannel>& channels)
|
||||
: fItems()
|
||||
, fNumItems(0)
|
||||
, fOffsetMap()
|
||||
{
|
||||
fNumItems = channels.size();
|
||||
fItems = new nn_pollfd[fNumItems];
|
||||
|
||||
for (int i = 0; i < fNumItems; ++i)
|
||||
{
|
||||
fItems[i].fd = static_cast<const FairMQSocketNN*>(&(channels.at(i).GetSocket()))->GetSocket();
|
||||
|
||||
int type = 0;
|
||||
size_t sz = sizeof(type);
|
||||
nn_getsockopt(static_cast<const FairMQSocketNN*>(&(channels.at(i).GetSocket()))->GetSocket(), NN_SOL_SOCKET, NN_PROTOCOL, &type, &sz);
|
||||
|
||||
SetItemEvents(fItems[i], type);
|
||||
}
|
||||
}
|
||||
|
||||
FairMQPollerNN::FairMQPollerNN(const vector<FairMQChannel*>& channels)
|
||||
: fItems()
|
||||
, fNumItems(0)
|
||||
, fOffsetMap()
|
||||
{
|
||||
fNumItems = channels.size();
|
||||
fItems = new nn_pollfd[fNumItems];
|
||||
|
||||
for (int i = 0; i < fNumItems; ++i)
|
||||
{
|
||||
fItems[i].fd = static_cast<const FairMQSocketNN*>(&(channels.at(i)->GetSocket()))->GetSocket();
|
||||
|
||||
int type = 0;
|
||||
size_t sz = sizeof(type);
|
||||
nn_getsockopt(static_cast<const FairMQSocketNN*>(&(channels.at(i)->GetSocket()))->GetSocket(), NN_SOL_SOCKET, NN_PROTOCOL, &type, &sz);
|
||||
|
||||
SetItemEvents(fItems[i], type);
|
||||
}
|
||||
}
|
||||
|
||||
FairMQPollerNN::FairMQPollerNN(const unordered_map<string, vector<FairMQChannel>>& channelsMap, const vector<string>& channelList)
|
||||
: fItems()
|
||||
, fNumItems(0)
|
||||
, fOffsetMap()
|
||||
{
|
||||
try
|
||||
{
|
||||
int offset = 0;
|
||||
// calculate offsets and the total size of the poll item set
|
||||
for (string channel : channelList)
|
||||
{
|
||||
fOffsetMap[channel] = offset;
|
||||
offset += channelsMap.at(channel).size();
|
||||
fNumItems += channelsMap.at(channel).size();
|
||||
}
|
||||
|
||||
fItems = new nn_pollfd[fNumItems];
|
||||
|
||||
int index = 0;
|
||||
for (string channel : channelList)
|
||||
{
|
||||
for (unsigned int i = 0; i < channelsMap.at(channel).size(); ++i)
|
||||
{
|
||||
index = fOffsetMap[channel] + i;
|
||||
fItems[index].fd = static_cast<const FairMQSocketNN*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket();
|
||||
|
||||
int type = 0;
|
||||
size_t sz = sizeof(type);
|
||||
nn_getsockopt(static_cast<const FairMQSocketNN*>(&(channelsMap.at(channel).at(i).GetSocket()))->GetSocket(), NN_SOL_SOCKET, NN_PROTOCOL, &type, &sz);
|
||||
|
||||
SetItemEvents(fItems[index], type);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (const std::out_of_range& oor)
|
||||
{
|
||||
LOG(error) << "at least one of the provided channel keys for poller initialization is invalid";
|
||||
LOG(error) << "out of range error: " << oor.what() << '\n';
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
void FairMQPollerNN::SetItemEvents(nn_pollfd& item, const int type)
|
||||
{
|
||||
if (type == NN_REQ || type == NN_REP || type == NN_PAIR)
|
||||
{
|
||||
item.events = NN_POLLIN|NN_POLLOUT;
|
||||
}
|
||||
else if (type == NN_PUSH || type == NN_PUB)
|
||||
{
|
||||
item.events = NN_POLLOUT;
|
||||
}
|
||||
else if (type == NN_PULL || type == NN_SUB)
|
||||
{
|
||||
item.events = NN_POLLIN;
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG(error) << "invalid poller configuration, exiting.";
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
void FairMQPollerNN::Poll(const int timeout)
|
||||
{
|
||||
if (nn_poll(fItems, fNumItems, timeout) < 0)
|
||||
{
|
||||
if (errno == ETERM)
|
||||
{
|
||||
LOG(debug) << "polling exited, reason: " << nn_strerror(errno);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG(error) << "polling failed, reason: " << nn_strerror(errno);
|
||||
throw std::runtime_error("polling failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool FairMQPollerNN::CheckInput(const int index)
|
||||
{
|
||||
if (fItems[index].revents & (NN_POLLIN | NN_POLLOUT))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool FairMQPollerNN::CheckOutput(const int index)
|
||||
{
|
||||
if (fItems[index].revents & NN_POLLOUT)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool FairMQPollerNN::CheckInput(const string& channelKey, const int index)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (fItems[fOffsetMap.at(channelKey) + index].revents & (NN_POLLIN | NN_POLLOUT))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
catch (const std::out_of_range& oor)
|
||||
{
|
||||
LOG(error) << "invalid channel key: \"" << channelKey << "\"";
|
||||
LOG(error) << "out of range error: " << oor.what() << '\n';
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
bool FairMQPollerNN::CheckOutput(const string& channelKey, const int index)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (fItems[fOffsetMap.at(channelKey) + index].revents & NN_POLLOUT)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
catch (const std::out_of_range& oor)
|
||||
{
|
||||
LOG(error) << "invalid channel key: \"" << channelKey << "\"";
|
||||
LOG(error) << "out of range error: " << oor.what() << '\n';
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
FairMQPollerNN::~FairMQPollerNN()
|
||||
{
|
||||
delete[] fItems;
|
||||
}
|
@@ -1,58 +0,0 @@
|
||||
/********************************************************************************
|
||||
* Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH *
|
||||
* *
|
||||
* This software is distributed under the terms of the *
|
||||
* GNU Lesser General Public Licence (LGPL) version 3, *
|
||||
* copied verbatim in the file "LICENSE" *
|
||||
********************************************************************************/
|
||||
/**
|
||||
* FairMQPollerNN.h
|
||||
*
|
||||
* @since 2014-01-23
|
||||
* @author A. Rybalchenko
|
||||
*/
|
||||
|
||||
#ifndef FAIRMQPOLLERNN_H_
|
||||
#define FAIRMQPOLLERNN_H_
|
||||
|
||||
#include <vector>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "FairMQPoller.h"
|
||||
#include "FairMQChannel.h"
|
||||
#include "FairMQTransportFactoryNN.h"
|
||||
|
||||
class FairMQChannel;
|
||||
struct nn_pollfd;
|
||||
|
||||
class FairMQPollerNN final : public FairMQPoller
|
||||
{
|
||||
friend class FairMQChannel;
|
||||
friend class FairMQTransportFactoryNN;
|
||||
|
||||
public:
|
||||
FairMQPollerNN(const std::vector<FairMQChannel>& channels);
|
||||
FairMQPollerNN(const std::vector<FairMQChannel*>& channels);
|
||||
FairMQPollerNN(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList);
|
||||
|
||||
FairMQPollerNN(const FairMQPollerNN&) = delete;
|
||||
FairMQPollerNN operator=(const FairMQPollerNN&) = delete;
|
||||
|
||||
void SetItemEvents(nn_pollfd& item, const int type);
|
||||
|
||||
void Poll(const int timeout) override;
|
||||
bool CheckInput(const int index) override;
|
||||
bool CheckOutput(const int index) override;
|
||||
bool CheckInput(const std::string& channelKey, const int index) override;
|
||||
bool CheckOutput(const std::string& channelKey, const int index) override;
|
||||
|
||||
~FairMQPollerNN() override;
|
||||
|
||||
private:
|
||||
nn_pollfd* fItems;
|
||||
int fNumItems;
|
||||
|
||||
std::unordered_map<std::string, int> fOffsetMap;
|
||||
};
|
||||
|
||||
#endif /* FAIRMQPOLLERNN_H_ */
|
@@ -1,628 +0,0 @@
|
||||
/********************************************************************************
|
||||
* Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH *
|
||||
* *
|
||||
* This software is distributed under the terms of the *
|
||||
* GNU Lesser General Public Licence (LGPL) version 3, *
|
||||
* copied verbatim in the file "LICENSE" *
|
||||
********************************************************************************/
|
||||
/**
|
||||
* FairMQSocketNN.cxx
|
||||
*
|
||||
* @since 2012-12-05
|
||||
* @author A. Rybalchenko
|
||||
*/
|
||||
|
||||
#include "FairMQSocketNN.h"
|
||||
#include "FairMQMessageNN.h"
|
||||
#include "FairMQLogger.h"
|
||||
#include "FairMQUnmanagedRegionNN.h"
|
||||
#include <fairmq/Tools.h>
|
||||
|
||||
#include <nanomsg/nn.h>
|
||||
#include <nanomsg/pipeline.h>
|
||||
#include <nanomsg/pubsub.h>
|
||||
#include <nanomsg/reqrep.h>
|
||||
#include <nanomsg/pair.h>
|
||||
|
||||
#include <sstream>
|
||||
#include <msgpack.hpp>
|
||||
|
||||
using namespace std;
|
||||
using namespace fair::mq;
|
||||
|
||||
atomic<bool> FairMQSocketNN::fInterrupted(false);
|
||||
|
||||
FairMQSocketNN::FairMQSocketNN(const string& type, const string& name, const string& id /*= ""*/, FairMQTransportFactory* fac /*=nullptr*/)
|
||||
: FairMQSocket{fac}
|
||||
, fSocket(-1)
|
||||
, fId(id + "." + name + "." + type)
|
||||
, fBytesTx(0)
|
||||
, fBytesRx(0)
|
||||
, fMessagesTx(0)
|
||||
, fMessagesRx(0)
|
||||
, fSndTimeout(100)
|
||||
, fRcvTimeout(100)
|
||||
, fLinger(500)
|
||||
{
|
||||
if (type == "router" || type == "dealer")
|
||||
{
|
||||
// Additional info about using the sockets ROUTER and DEALER with nanomsg can be found in:
|
||||
// http://250bpm.com/blog:14
|
||||
// http://www.freelists.org/post/nanomsg/a-stupid-load-balancing-question,1
|
||||
fSocket = nn_socket(AF_SP_RAW, GetConstant(type));
|
||||
if (fSocket == -1)
|
||||
{
|
||||
LOG(error) << "failed creating socket " << fId << ", reason: " << nn_strerror(errno);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
fSocket = nn_socket(AF_SP, GetConstant(type));
|
||||
if (fSocket == -1)
|
||||
{
|
||||
LOG(error) << "failed creating socket " << fId << ", reason: " << nn_strerror(errno);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if (type == "sub")
|
||||
{
|
||||
nn_setsockopt(fSocket, NN_SUB, NN_SUB_SUBSCRIBE, nullptr, 0);
|
||||
}
|
||||
}
|
||||
|
||||
if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_SNDTIMEO, &fSndTimeout, sizeof(fSndTimeout)) != 0)
|
||||
{
|
||||
LOG(error) << "Failed setting NN_SNDTIMEO socket option, reason: " << nn_strerror(errno);
|
||||
}
|
||||
|
||||
if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_RCVTIMEO, &fRcvTimeout, sizeof(fRcvTimeout)) != 0)
|
||||
{
|
||||
LOG(error) << "Failed setting NN_RCVTIMEO socket option, reason: " << nn_strerror(errno);
|
||||
}
|
||||
|
||||
#ifdef NN_RCVMAXSIZE
|
||||
int rcvSize = -1;
|
||||
if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_RCVMAXSIZE, &rcvSize, sizeof(rcvSize)) != 0)
|
||||
{
|
||||
LOG(error) << "Failed setting NN_RCVMAXSIZE socket option, reason: " << nn_strerror(errno);
|
||||
}
|
||||
#endif
|
||||
|
||||
LOG(debug) << "Created socket " << GetId();
|
||||
}
|
||||
|
||||
bool FairMQSocketNN::Bind(const string& address)
|
||||
{
|
||||
// LOG(info) << "bind socket " << fId << " on " << address;
|
||||
|
||||
if (nn_bind(fSocket, address.c_str()) < 0)
|
||||
{
|
||||
LOG(error) << "failed binding socket " << fId << ", reason: " << nn_strerror(errno);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool FairMQSocketNN::Connect(const string& address)
|
||||
{
|
||||
// LOG(info) << "connect socket " << fId << " to " << address;
|
||||
|
||||
if (nn_connect(fSocket, address.c_str()) < 0)
|
||||
{
|
||||
LOG(error) << "failed connecting socket " << fId << ", reason: " << nn_strerror(errno);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
int FairMQSocketNN::Send(FairMQMessagePtr& msg, const int timeout)
|
||||
{
|
||||
int flags = 0;
|
||||
if (timeout == 0)
|
||||
{
|
||||
flags = NN_DONTWAIT;
|
||||
}
|
||||
int nbytes = -1;
|
||||
int elapsed = 0;
|
||||
|
||||
FairMQMessageNN* msgPtr = static_cast<FairMQMessageNN*>(msg.get());
|
||||
void* bufPtr = msgPtr->GetMessage();
|
||||
|
||||
while (true)
|
||||
{
|
||||
if (msgPtr->fRegionPtr == nullptr)
|
||||
{
|
||||
nbytes = nn_send(fSocket, &bufPtr, NN_MSG, flags);
|
||||
}
|
||||
else
|
||||
{
|
||||
nbytes = nn_send(fSocket, bufPtr, msg->GetSize(), flags);
|
||||
// nn_send copies the data, safe to call region callback here
|
||||
static_cast<FairMQUnmanagedRegionNN*>(msgPtr->fRegionPtr)->fCallback(bufPtr, msg->GetSize(), reinterpret_cast<void*>(msgPtr->fHint));
|
||||
}
|
||||
|
||||
if (nbytes >= 0)
|
||||
{
|
||||
fBytesTx += nbytes;
|
||||
++fMessagesTx;
|
||||
static_cast<FairMQMessageNN*>(msg.get())->fReceiving = false;
|
||||
|
||||
return nbytes;
|
||||
}
|
||||
else if (nn_errno() == ETIMEDOUT)
|
||||
{
|
||||
if (!fInterrupted && ((flags & NN_DONTWAIT) == 0))
|
||||
{
|
||||
if (timeout > 0)
|
||||
{
|
||||
elapsed += fSndTimeout;
|
||||
if (elapsed >= timeout)
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
}
|
||||
else if (nn_errno() == EAGAIN)
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
else if (nn_errno() == ETERM)
|
||||
{
|
||||
LOG(info) << "terminating socket " << fId;
|
||||
return -1;
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG(error) << "Failed sending on socket " << fId << ", reason: " << nn_strerror(errno);
|
||||
return nbytes;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int FairMQSocketNN::Receive(FairMQMessagePtr& msg, const int timeout)
|
||||
{
|
||||
int flags = 0;
|
||||
if (timeout == 0)
|
||||
{
|
||||
flags = NN_DONTWAIT;
|
||||
}
|
||||
int elapsed = 0;
|
||||
|
||||
FairMQMessageNN* msgPtr = static_cast<FairMQMessageNN*>(msg.get());
|
||||
|
||||
while (true)
|
||||
{
|
||||
void* ptr = nullptr;
|
||||
int nbytes = nn_recv(fSocket, &ptr, NN_MSG, flags);
|
||||
if (nbytes >= 0)
|
||||
{
|
||||
fBytesRx += nbytes;
|
||||
++fMessagesRx;
|
||||
msgPtr->SetMessage(ptr, nbytes);
|
||||
msgPtr->fReceiving = true;
|
||||
return nbytes;
|
||||
}
|
||||
else if (nn_errno() == ETIMEDOUT)
|
||||
{
|
||||
if (!fInterrupted && ((flags & NN_DONTWAIT) == 0))
|
||||
{
|
||||
if (timeout > 0)
|
||||
{
|
||||
elapsed += fRcvTimeout;
|
||||
if (elapsed >= timeout)
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
}
|
||||
else if (nn_errno() == EAGAIN)
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
else if (nn_errno() == ETERM)
|
||||
{
|
||||
LOG(info) << "terminating socket " << fId;
|
||||
return -1;
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG(error) << "Failed receiving on socket " << fId << ", reason: " << nn_strerror(errno);
|
||||
return nbytes;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int64_t FairMQSocketNN::Send(vector<FairMQMessagePtr>& msgVec, const int timeout)
|
||||
{
|
||||
int flags = 0;
|
||||
if (timeout == 0)
|
||||
{
|
||||
flags = NN_DONTWAIT;
|
||||
}
|
||||
const unsigned int vecSize = msgVec.size();
|
||||
int elapsed = 0;
|
||||
|
||||
// create msgpack simple buffer
|
||||
msgpack::sbuffer sbuf;
|
||||
// create msgpack packer
|
||||
msgpack::packer<msgpack::sbuffer> packer(&sbuf);
|
||||
|
||||
// pack all parts into a single msgpack simple buffer
|
||||
for (unsigned int i = 0; i < vecSize; ++i)
|
||||
{
|
||||
FairMQMessageNN* partPtr = static_cast<FairMQMessageNN*>(msgVec[i].get());
|
||||
|
||||
partPtr->fReceiving = false;
|
||||
packer.pack_bin(msgVec[i]->GetSize());
|
||||
packer.pack_bin_body(static_cast<char*>(msgVec[i]->GetData()), msgVec[i]->GetSize());
|
||||
// call region callback
|
||||
if (partPtr->fRegionPtr)
|
||||
{
|
||||
static_cast<FairMQUnmanagedRegionNN*>(partPtr->fRegionPtr)->fCallback(partPtr->GetMessage(), partPtr->GetSize(), reinterpret_cast<void*>(partPtr->fHint));
|
||||
}
|
||||
}
|
||||
|
||||
while (true)
|
||||
{
|
||||
int64_t nbytes = nn_send(fSocket, sbuf.data(), sbuf.size(), flags);
|
||||
if (nbytes >= 0)
|
||||
{
|
||||
fBytesTx += nbytes;
|
||||
++fMessagesTx;
|
||||
return nbytes;
|
||||
}
|
||||
else if (nn_errno() == ETIMEDOUT)
|
||||
{
|
||||
if (!fInterrupted && ((flags & NN_DONTWAIT) == 0))
|
||||
{
|
||||
if (timeout > 0)
|
||||
{
|
||||
elapsed += fSndTimeout;
|
||||
if (elapsed >= timeout)
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
}
|
||||
else if (nn_errno() == EAGAIN)
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
else if (nn_errno() == ETERM)
|
||||
{
|
||||
LOG(info) << "terminating socket " << fId;
|
||||
return -1;
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG(error) << "Failed sending on socket " << fId << ", reason: " << nn_strerror(errno);
|
||||
return nbytes;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int64_t FairMQSocketNN::Receive(vector<FairMQMessagePtr>& msgVec, const int timeout)
|
||||
{
|
||||
int flags = 0;
|
||||
if (timeout == 0)
|
||||
{
|
||||
flags = NN_DONTWAIT;
|
||||
}
|
||||
// Warn if the vector is filled before Receive() and empty it.
|
||||
// if (msgVec.size() > 0)
|
||||
// {
|
||||
// LOG(warn) << "Message vector contains elements before Receive(), they will be deleted!";
|
||||
// msgVec.clear();
|
||||
// }
|
||||
|
||||
int elapsed = 0;
|
||||
|
||||
while (true)
|
||||
{
|
||||
// pointer to point to received message buffer
|
||||
char* ptr = nullptr;
|
||||
// receive the message into a buffer allocated by nanomsg and let ptr point to it
|
||||
int nbytes = nn_recv(fSocket, &ptr, NN_MSG, flags);
|
||||
if (nbytes >= 0) // if no errors or non-blocking timeouts
|
||||
{
|
||||
// store statistics on how many bytes received
|
||||
fBytesRx += nbytes;
|
||||
// store statistics on how many messages received (count messages instead of parts)
|
||||
++fMessagesRx;
|
||||
|
||||
// offset to be used by msgpack to handle separate chunks
|
||||
size_t offset = 0;
|
||||
while (offset != static_cast<size_t>(nbytes)) // continue until all parts have been read
|
||||
{
|
||||
// vector of chars to hold blob (unlike char*/void* this type can be converted to by msgpack)
|
||||
vector<char> buf;
|
||||
|
||||
// unpack and convert chunk
|
||||
msgpack::unpacked result;
|
||||
unpack(result, ptr, nbytes, offset);
|
||||
msgpack::object object(result.get());
|
||||
object.convert(buf);
|
||||
// get the single message size
|
||||
size_t size = buf.size() * sizeof(char);
|
||||
FairMQMessagePtr part(new FairMQMessageNN(size, GetTransport()));
|
||||
static_cast<FairMQMessageNN*>(part.get())->fReceiving = true;
|
||||
memcpy(part->GetData(), buf.data(), size);
|
||||
msgVec.push_back(move(part));
|
||||
}
|
||||
|
||||
nn_freemsg(ptr);
|
||||
return nbytes;
|
||||
}
|
||||
else if (nn_errno() == ETIMEDOUT)
|
||||
{
|
||||
if (!fInterrupted && ((flags & NN_DONTWAIT) == 0))
|
||||
{
|
||||
if (timeout > 0)
|
||||
{
|
||||
elapsed += fRcvTimeout;
|
||||
if (elapsed >= timeout)
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
}
|
||||
else if (nn_errno() == EAGAIN)
|
||||
{
|
||||
return -2;
|
||||
}
|
||||
else if (nn_errno() == ETERM)
|
||||
{
|
||||
LOG(info) << "terminating socket " << fId;
|
||||
return -1;
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG(error) << "Failed receiving on socket " << fId << ", reason: " << nn_strerror(errno);
|
||||
return nbytes;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void FairMQSocketNN::Close()
|
||||
{
|
||||
nn_close(fSocket);
|
||||
}
|
||||
|
||||
void FairMQSocketNN::Interrupt()
|
||||
{
|
||||
fInterrupted = true;
|
||||
}
|
||||
|
||||
void FairMQSocketNN::Resume()
|
||||
{
|
||||
fInterrupted = false;
|
||||
}
|
||||
|
||||
int FairMQSocketNN::GetSocket() const
|
||||
{
|
||||
return fSocket;
|
||||
}
|
||||
|
||||
void FairMQSocketNN::SetOption(const string& option, const void* value, size_t valueSize)
|
||||
{
|
||||
if (option == "snd-size" || option == "rcv-size")
|
||||
{
|
||||
int val = *(static_cast<int*>(const_cast<void*>(value)));
|
||||
if (val <= 0)
|
||||
{
|
||||
LOG(warn) << "value for sndKernelSize/rcvKernelSize should be greater than 0, leaving unchanged.";
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (option == "snd-hwm" || option == "rcv-hwm")
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (option == "linger")
|
||||
{
|
||||
fLinger = *static_cast<int*>(const_cast<void*>(value));
|
||||
return;
|
||||
}
|
||||
|
||||
int rc = nn_setsockopt(fSocket, NN_SOL_SOCKET, GetConstant(option), value, valueSize);
|
||||
if (rc < 0)
|
||||
{
|
||||
LOG(error) << "failed setting socket option, reason: " << nn_strerror(errno);
|
||||
}
|
||||
}
|
||||
|
||||
void FairMQSocketNN::GetOption(const string& option, void* value, size_t* valueSize)
|
||||
{
|
||||
if (option == "linger")
|
||||
{
|
||||
*static_cast<int*>(value) = fLinger;
|
||||
return;
|
||||
}
|
||||
|
||||
if (option == "snd-hwm" || option == "rcv-hwm")
|
||||
{
|
||||
*static_cast<int*>(value) = -1;
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
int rc = nn_getsockopt(fSocket, NN_SOL_SOCKET, GetConstant(option), value, valueSize);
|
||||
if (rc < 0)
|
||||
{
|
||||
LOG(error) << "failed getting socket option, reason: " << nn_strerror(errno);
|
||||
}
|
||||
}
|
||||
|
||||
void FairMQSocketNN::SetLinger(const int value)
|
||||
{
|
||||
fLinger = value;
|
||||
}
|
||||
|
||||
int FairMQSocketNN::GetLinger() const
|
||||
{
|
||||
return fLinger;
|
||||
}
|
||||
|
||||
void FairMQSocketNN::SetSndBufSize(const int /* value */)
|
||||
{
|
||||
// not used in nanomsg
|
||||
}
|
||||
|
||||
int FairMQSocketNN::GetSndBufSize() const
|
||||
{
|
||||
// not used in nanomsg
|
||||
return -1;
|
||||
}
|
||||
|
||||
void FairMQSocketNN::SetRcvBufSize(const int /* value */)
|
||||
{
|
||||
// not used in nanomsg
|
||||
}
|
||||
|
||||
int FairMQSocketNN::GetRcvBufSize() const
|
||||
{
|
||||
// not used in nanomsg
|
||||
return -1;
|
||||
}
|
||||
|
||||
void FairMQSocketNN::SetSndKernelSize(const int value)
|
||||
{
|
||||
if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_SNDBUF, &value, sizeof(value)) < 0) {
|
||||
throw SocketError(tools::ToString("failed setting NN_SNDBUF, reason: ", nn_strerror(errno)));
|
||||
}
|
||||
}
|
||||
|
||||
int FairMQSocketNN::GetSndKernelSize() const
|
||||
{
|
||||
int value = 0;
|
||||
size_t valueSize;
|
||||
if (nn_getsockopt(fSocket, NN_SOL_SOCKET, NN_SNDBUF, &value, &valueSize) < 0) {
|
||||
throw SocketError(tools::ToString("failed getting NN_SNDBUF, reason: ", nn_strerror(errno)));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
void FairMQSocketNN::SetRcvKernelSize(const int value)
|
||||
{
|
||||
if (nn_setsockopt(fSocket, NN_SOL_SOCKET, NN_RCVBUF, &value, sizeof(value)) < 0) {
|
||||
throw SocketError(tools::ToString("failed setting NN_RCVBUF, reason: ", nn_strerror(errno)));
|
||||
}
|
||||
}
|
||||
|
||||
int FairMQSocketNN::GetRcvKernelSize() const
|
||||
{
|
||||
int value = 0;
|
||||
size_t valueSize;
|
||||
if (nn_getsockopt(fSocket, NN_SOL_SOCKET, NN_RCVBUF, &value, &valueSize) < 0) {
|
||||
throw SocketError(tools::ToString("failed getting NN_RCVBUF, reason: ", nn_strerror(errno)));
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
|
||||
unsigned long FairMQSocketNN::GetBytesTx() const
|
||||
{
|
||||
return fBytesTx;
|
||||
}
|
||||
|
||||
unsigned long FairMQSocketNN::GetBytesRx() const
|
||||
{
|
||||
return fBytesRx;
|
||||
}
|
||||
|
||||
unsigned long FairMQSocketNN::GetMessagesTx() const
|
||||
{
|
||||
return fMessagesTx;
|
||||
}
|
||||
|
||||
unsigned long FairMQSocketNN::GetMessagesRx() const
|
||||
{
|
||||
return fMessagesRx;
|
||||
}
|
||||
|
||||
int FairMQSocketNN::GetConstant(const string& constant)
|
||||
{
|
||||
if (constant == "")
|
||||
return 0;
|
||||
if (constant == "sub")
|
||||
return NN_SUB;
|
||||
if (constant == "pub")
|
||||
return NN_PUB;
|
||||
if (constant == "xsub")
|
||||
return NN_SUB;
|
||||
if (constant == "xpub")
|
||||
return NN_PUB;
|
||||
if (constant == "push")
|
||||
return NN_PUSH;
|
||||
if (constant == "pull")
|
||||
return NN_PULL;
|
||||
if (constant == "req")
|
||||
return NN_REQ;
|
||||
if (constant == "rep")
|
||||
return NN_REP;
|
||||
if (constant == "dealer")
|
||||
return NN_REQ;
|
||||
if (constant == "router")
|
||||
return NN_REP;
|
||||
if (constant == "pair")
|
||||
return NN_PAIR;
|
||||
|
||||
if (constant == "snd-hwm")
|
||||
return NN_SNDBUF;
|
||||
if (constant == "rcv-hwm")
|
||||
return NN_RCVBUF;
|
||||
if (constant == "snd-size")
|
||||
return NN_SNDBUF;
|
||||
if (constant == "rcv-size")
|
||||
return NN_RCVBUF;
|
||||
if (constant == "snd-more")
|
||||
{
|
||||
LOG(error) << "Multipart messages functionality currently not supported by nanomsg!";
|
||||
return -1;
|
||||
}
|
||||
if (constant == "rcv-more")
|
||||
{
|
||||
LOG(error) << "Multipart messages functionality currently not supported by nanomsg!";
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (constant == "linger")
|
||||
return NN_LINGER;
|
||||
if (constant == "no-block")
|
||||
return NN_DONTWAIT;
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
FairMQSocketNN::~FairMQSocketNN()
|
||||
{
|
||||
Close();
|
||||
}
|
@@ -1,81 +0,0 @@
|
||||
/********************************************************************************
|
||||
* Copyright (C) 2014-2018 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH *
|
||||
* *
|
||||
* This software is distributed under the terms of the *
|
||||
* GNU Lesser General Public Licence (LGPL) version 3, *
|
||||
* copied verbatim in the file "LICENSE" *
|
||||
********************************************************************************/
|
||||
|
||||
#ifndef FAIRMQSOCKETNN_H_
|
||||
#define FAIRMQSOCKETNN_H_
|
||||
|
||||
#include <vector>
|
||||
#include <atomic>
|
||||
|
||||
#include "FairMQSocket.h"
|
||||
#include "FairMQMessage.h"
|
||||
class FairMQTransportFactory;
|
||||
|
||||
class FairMQSocketNN final : public FairMQSocket
|
||||
{
|
||||
public:
|
||||
FairMQSocketNN(const std::string& type, const std::string& name, const std::string& id = "", FairMQTransportFactory* fac = nullptr);
|
||||
FairMQSocketNN(const FairMQSocketNN&) = delete;
|
||||
FairMQSocketNN operator=(const FairMQSocketNN&) = delete;
|
||||
|
||||
std::string GetId() const override { return fId; }
|
||||
|
||||
bool Bind(const std::string& address) override;
|
||||
bool Connect(const std::string& address) override;
|
||||
|
||||
int Send(FairMQMessagePtr& msg, const int timeout = -1) override;
|
||||
int Receive(FairMQMessagePtr& msg, const int timeout = -1) override;
|
||||
int64_t Send(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int timeout = -1) override;
|
||||
int64_t Receive(std::vector<std::unique_ptr<FairMQMessage>>& msgVec, const int timeout = -1) override;
|
||||
|
||||
int GetSocket() const;
|
||||
|
||||
void Close() override;
|
||||
|
||||
static void Interrupt();
|
||||
static void Resume();
|
||||
|
||||
void SetOption(const std::string& option, const void* value, size_t valueSize) override;
|
||||
void GetOption(const std::string& option, void* value, size_t* valueSize) override;
|
||||
|
||||
void SetLinger(const int value) override;
|
||||
int GetLinger() const override;
|
||||
void SetSndBufSize(const int value) override;
|
||||
int GetSndBufSize() const override;
|
||||
void SetRcvBufSize(const int value) override;
|
||||
int GetRcvBufSize() const override;
|
||||
void SetSndKernelSize(const int value) override;
|
||||
int GetSndKernelSize() const override;
|
||||
void SetRcvKernelSize(const int value) override;
|
||||
int GetRcvKernelSize() const override;
|
||||
|
||||
unsigned long GetBytesTx() const override;
|
||||
unsigned long GetBytesRx() const override;
|
||||
unsigned long GetMessagesTx() const override;
|
||||
unsigned long GetMessagesRx() const override;
|
||||
|
||||
static int GetConstant(const std::string& constant);
|
||||
|
||||
~FairMQSocketNN() override;
|
||||
|
||||
private:
|
||||
int fSocket;
|
||||
std::string fId;
|
||||
std::atomic<unsigned long> fBytesTx;
|
||||
std::atomic<unsigned long> fBytesRx;
|
||||
std::atomic<unsigned long> fMessagesTx;
|
||||
std::atomic<unsigned long> fMessagesRx;
|
||||
|
||||
static std::atomic<bool> fInterrupted;
|
||||
|
||||
int fSndTimeout;
|
||||
int fRcvTimeout;
|
||||
int fLinger;
|
||||
};
|
||||
|
||||
#endif /* FAIRMQSOCKETNN_H_ */
|
@@ -1,100 +0,0 @@
|
||||
/********************************************************************************
|
||||
* Copyright (C) 2014-2017 GSI Helmholtzzentrum fuer Schwerionenforschung Gmb *
|
||||
* *
|
||||
* This software is distributed under the terms of the *
|
||||
* GNU Lesser General Public Licence (LGPL) version 3, *
|
||||
* copied verbatim in the file "LICENSE" *
|
||||
********************************************************************************/
|
||||
|
||||
#include "FairMQTransportFactoryNN.h"
|
||||
|
||||
#include <nanomsg/nn.h>
|
||||
#include <algorithm>
|
||||
#include <thread>
|
||||
#include <chrono>
|
||||
|
||||
using namespace std;
|
||||
|
||||
fair::mq::Transport FairMQTransportFactoryNN::fTransportType = fair::mq::Transport::NN;
|
||||
|
||||
FairMQTransportFactoryNN::FairMQTransportFactoryNN(const string& id, const fair::mq::ProgOptions* /*config*/)
|
||||
: FairMQTransportFactory(id)
|
||||
, fRegionCounter(0)
|
||||
{
|
||||
LOG(debug) << "Transport: Using nanomsg library";
|
||||
}
|
||||
|
||||
FairMQMessagePtr FairMQTransportFactoryNN::CreateMessage()
|
||||
{
|
||||
return unique_ptr<FairMQMessage>(new FairMQMessageNN(this));
|
||||
}
|
||||
|
||||
FairMQMessagePtr FairMQTransportFactoryNN::CreateMessage(const size_t size)
|
||||
{
|
||||
return unique_ptr<FairMQMessage>(new FairMQMessageNN(size, this));
|
||||
}
|
||||
|
||||
FairMQMessagePtr FairMQTransportFactoryNN::CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint)
|
||||
{
|
||||
return unique_ptr<FairMQMessage>(new FairMQMessageNN(data, size, ffn, hint, this));
|
||||
}
|
||||
|
||||
FairMQMessagePtr FairMQTransportFactoryNN::CreateMessage(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint)
|
||||
{
|
||||
return unique_ptr<FairMQMessage>(new FairMQMessageNN(region, data, size, hint, this));
|
||||
}
|
||||
|
||||
FairMQSocketPtr FairMQTransportFactoryNN::CreateSocket(const string& type, const string& name)
|
||||
{
|
||||
unique_ptr<FairMQSocket> socket(new FairMQSocketNN(type, name, GetId(), this));
|
||||
fSockets.push_back(socket.get());
|
||||
return socket;
|
||||
}
|
||||
|
||||
FairMQPollerPtr FairMQTransportFactoryNN::CreatePoller(const vector<FairMQChannel>& channels) const
|
||||
{
|
||||
return unique_ptr<FairMQPoller>(new FairMQPollerNN(channels));
|
||||
}
|
||||
|
||||
FairMQPollerPtr FairMQTransportFactoryNN::CreatePoller(const std::vector<FairMQChannel*>& channels) const
|
||||
{
|
||||
return unique_ptr<FairMQPoller>(new FairMQPollerNN(channels));
|
||||
}
|
||||
|
||||
FairMQPollerPtr FairMQTransportFactoryNN::CreatePoller(const unordered_map<string, vector<FairMQChannel>>& channelsMap, const vector<string>& channelList) const
|
||||
{
|
||||
return unique_ptr<FairMQPoller>(new FairMQPollerNN(channelsMap, channelList));
|
||||
}
|
||||
|
||||
FairMQUnmanagedRegionPtr FairMQTransportFactoryNN::CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback, const std::string& path /* = "" */, int flags /* = 0 */)
|
||||
{
|
||||
return unique_ptr<FairMQUnmanagedRegion>(new FairMQUnmanagedRegionNN(++fRegionCounter, size, callback, path, flags, this));
|
||||
}
|
||||
|
||||
FairMQUnmanagedRegionPtr FairMQTransportFactoryNN::CreateUnmanagedRegion(const size_t size, const int64_t userFlags, FairMQRegionCallback callback, const std::string& path /* = "" */, int flags /* = 0 */)
|
||||
{
|
||||
return unique_ptr<FairMQUnmanagedRegion>(new FairMQUnmanagedRegionNN(size, userFlags, callback, path, flags, this));
|
||||
}
|
||||
|
||||
fair::mq::Transport FairMQTransportFactoryNN::GetType() const
|
||||
{
|
||||
return fTransportType;
|
||||
}
|
||||
|
||||
void FairMQTransportFactoryNN::Reset()
|
||||
{
|
||||
auto it = max_element(fSockets.begin(), fSockets.end(), [](FairMQSocket* s1, FairMQSocket* s2) {
|
||||
return static_cast<FairMQSocketNN*>(s1)->GetLinger() < static_cast<FairMQSocketNN*>(s2)->GetLinger();
|
||||
});
|
||||
if (it != fSockets.end()) {
|
||||
this_thread::sleep_for(chrono::milliseconds(static_cast<FairMQSocketNN*>(*it)->GetLinger()));
|
||||
}
|
||||
fSockets.clear();
|
||||
}
|
||||
|
||||
FairMQTransportFactoryNN::~FairMQTransportFactoryNN()
|
||||
{
|
||||
LOG(debug) << "Destroying Shared Memory transport...";
|
||||
// nn_term();
|
||||
// see https://www.freelists.org/post/nanomsg/Getting-rid-of-nn-init-and-nn-term,8
|
||||
}
|
@@ -1,59 +0,0 @@
|
||||
/********************************************************************************
|
||||
* Copyright (C) 2014-2017 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH *
|
||||
* *
|
||||
* This software is distributed under the terms of the *
|
||||
* GNU Lesser General Public Licence (LGPL) version 3, *
|
||||
* copied verbatim in the file "LICENSE" *
|
||||
********************************************************************************/
|
||||
|
||||
#ifndef FAIRMQTRANSPORTFACTORYNN_H_
|
||||
#define FAIRMQTRANSPORTFACTORYNN_H_
|
||||
|
||||
#include "FairMQTransportFactory.h"
|
||||
#include "FairMQMessageNN.h"
|
||||
#include "FairMQSocketNN.h"
|
||||
#include "FairMQPollerNN.h"
|
||||
#include "FairMQUnmanagedRegionNN.h"
|
||||
#include <fairmq/ProgOptions.h>
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
class FairMQTransportFactoryNN final : public FairMQTransportFactory
|
||||
{
|
||||
public:
|
||||
FairMQTransportFactoryNN(const std::string& id = "", const fair::mq::ProgOptions* config = nullptr);
|
||||
~FairMQTransportFactoryNN() override;
|
||||
|
||||
FairMQMessagePtr CreateMessage() override;
|
||||
FairMQMessagePtr CreateMessage(const size_t size) override;
|
||||
FairMQMessagePtr CreateMessage(void* data, const size_t size, fairmq_free_fn* ffn, void* hint = nullptr) override;
|
||||
FairMQMessagePtr CreateMessage(FairMQUnmanagedRegionPtr& region, void* data, const size_t size, void* hint = 0) override;
|
||||
|
||||
FairMQSocketPtr CreateSocket(const std::string& type, const std::string& name) override;
|
||||
|
||||
FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel>& channels) const override;
|
||||
FairMQPollerPtr CreatePoller(const std::vector<FairMQChannel*>& channels) const override;
|
||||
FairMQPollerPtr CreatePoller(const std::unordered_map<std::string, std::vector<FairMQChannel>>& channelsMap, const std::vector<std::string>& channelList) const override;
|
||||
|
||||
FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, FairMQRegionCallback callback, const std::string& path = "", int flags = 0) override;
|
||||
FairMQUnmanagedRegionPtr CreateUnmanagedRegion(const size_t size, int64_t userFlags, FairMQRegionCallback callback = nullptr, const std::string& path = "", int flags = 0) override;
|
||||
|
||||
void SubscribeToRegionEvents(FairMQRegionEventCallback /* callback */) override { LOG(error) << "SubscribeToRegionEvents not yet implemented for nanomsg"; }
|
||||
bool SubscribedToRegionEvents() override { LOG(error) << "Region event subscriptions not yet implemented for nanomsg"; return false; }
|
||||
void UnsubscribeFromRegionEvents() override { LOG(error) << "UnsubscribeFromRegionEvents not yet implemented for nanomsg"; }
|
||||
std::vector<FairMQRegionInfo> GetRegionInfo() override { LOG(error) << "GetRegionInfo not yet implemented for nanomsg, returning empty vector"; return std::vector<FairMQRegionInfo>(); }
|
||||
|
||||
fair::mq::Transport GetType() const override;
|
||||
|
||||
void Interrupt() override { FairMQSocketNN::Interrupt(); }
|
||||
void Resume() override { FairMQSocketNN::Resume(); }
|
||||
void Reset() override;
|
||||
|
||||
private:
|
||||
static fair::mq::Transport fTransportType;
|
||||
uint64_t fRegionCounter;
|
||||
mutable std::vector<FairMQSocket*> fSockets;
|
||||
};
|
||||
|
||||
#endif /* FAIRMQTRANSPORTFACTORYNN_H_ */
|
@@ -1,52 +0,0 @@
|
||||
/********************************************************************************
|
||||
* Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH *
|
||||
* *
|
||||
* This software is distributed under the terms of the *
|
||||
* GNU Lesser General Public Licence (LGPL) version 3, *
|
||||
* copied verbatim in the file "LICENSE" *
|
||||
********************************************************************************/
|
||||
|
||||
#include "FairMQUnmanagedRegionNN.h"
|
||||
#include "FairMQLogger.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
FairMQUnmanagedRegionNN::FairMQUnmanagedRegionNN(uint64_t id, const size_t size, FairMQRegionCallback callback, const std::string& /*path = "" */, int /*flags = 0 */, FairMQTransportFactory* factory /* = nullptr */)
|
||||
: FairMQUnmanagedRegion(factory)
|
||||
, fId(id)
|
||||
, fBuffer(malloc(size))
|
||||
, fSize(size)
|
||||
, fCallback(callback)
|
||||
{
|
||||
}
|
||||
|
||||
FairMQUnmanagedRegionNN::FairMQUnmanagedRegionNN(uint64_t id, const size_t size, const int64_t /*userFlags*/, FairMQRegionCallback callback, const std::string& /*path = "" */, int /*flags = 0 */, FairMQTransportFactory* factory /* = nullptr */)
|
||||
: FairMQUnmanagedRegion(factory)
|
||||
, fId(id)
|
||||
, fBuffer(malloc(size))
|
||||
, fSize(size)
|
||||
, fCallback(callback)
|
||||
{
|
||||
}
|
||||
|
||||
void* FairMQUnmanagedRegionNN::GetData() const
|
||||
{
|
||||
return fBuffer;
|
||||
}
|
||||
|
||||
size_t FairMQUnmanagedRegionNN::GetSize() const
|
||||
{
|
||||
return fSize;
|
||||
}
|
||||
|
||||
uint64_t FairMQUnmanagedRegionNN::GetId() const
|
||||
{
|
||||
return fId;
|
||||
}
|
||||
|
||||
|
||||
FairMQUnmanagedRegionNN::~FairMQUnmanagedRegionNN()
|
||||
{
|
||||
LOG(debug) << "destroying region";
|
||||
free(fBuffer);
|
||||
}
|
@@ -1,41 +0,0 @@
|
||||
/********************************************************************************
|
||||
* Copyright (C) 2014 GSI Helmholtzzentrum fuer Schwerionenforschung GmbH *
|
||||
* *
|
||||
* This software is distributed under the terms of the *
|
||||
* GNU Lesser General Public Licence (LGPL) version 3, *
|
||||
* copied verbatim in the file "LICENSE" *
|
||||
********************************************************************************/
|
||||
|
||||
#ifndef FAIRMQUNMANAGEDREGIONNN_H_
|
||||
#define FAIRMQUNMANAGEDREGIONNN_H_
|
||||
|
||||
#include "FairMQUnmanagedRegion.h"
|
||||
|
||||
#include <cstddef> // size_t
|
||||
#include <string>
|
||||
|
||||
class FairMQUnmanagedRegionNN final : public FairMQUnmanagedRegion
|
||||
{
|
||||
friend class FairMQSocketNN;
|
||||
|
||||
public:
|
||||
FairMQUnmanagedRegionNN(uint64_t id, const size_t size, FairMQRegionCallback callback, const std::string& path = "", int flags = 0, FairMQTransportFactory* factory = nullptr);
|
||||
FairMQUnmanagedRegionNN(uint64_t id, const size_t size, const int64_t userFlags, FairMQRegionCallback callback, const std::string& path = "", int flags = 0, FairMQTransportFactory* factory = nullptr);
|
||||
|
||||
FairMQUnmanagedRegionNN(const FairMQUnmanagedRegionNN&) = delete;
|
||||
FairMQUnmanagedRegionNN operator=(const FairMQUnmanagedRegionNN&) = delete;
|
||||
|
||||
void* GetData() const override;
|
||||
size_t GetSize() const override;
|
||||
uint64_t GetId() const override;
|
||||
|
||||
virtual ~FairMQUnmanagedRegionNN();
|
||||
|
||||
private:
|
||||
uint64_t fId;
|
||||
void* fBuffer;
|
||||
size_t fSize;
|
||||
FairMQRegionCallback fCallback;
|
||||
};
|
||||
|
||||
#endif /* FAIRMQUNMANAGEDREGIONNN_H_ */
|
@@ -63,7 +63,7 @@ Plugin::ProgOptions ConfigPluginProgramOptions()
|
||||
pluginOptions.add_options()
|
||||
("id", po::value<string >()->default_value(""), "Device ID.")
|
||||
("io-threads", po::value<int >()->default_value(1), "Number of I/O threads.")
|
||||
("transport", po::value<string >()->default_value("zeromq"), "Transport ('zeromq'/'nanomsg'/'shmem').")
|
||||
("transport", po::value<string >()->default_value("zeromq"), "Transport ('zeromq'/'shmem').")
|
||||
("network-interface", po::value<string >()->default_value("default"), "Network interface to bind on (e.g. eth0, ib0..., default will try to detect the interface of the default route).")
|
||||
("init-timeout", po::value<int >()->default_value(120), "Timeout for the initialization in seconds (when expecting dynamic initialization).")
|
||||
("max-run-time", po::value<uint64_t >()->default_value(0), "Maximum runtime for the Running state handler, after which state will change to Ready (in seconds, 0 for no limit).")
|
||||
|
@@ -59,7 +59,7 @@ else
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Usage: startBenchmark [message size=1000000] [number of iterations=0] [transport=zeromq/nanomsg/shmem] [affinity=false]"
|
||||
echo "Usage: startBenchmark [message size=1000000] [number of iterations=0] [transport=zeromq/shmem] [affinity=false]"
|
||||
|
||||
SAMPLER="fairmq-bsampler"
|
||||
SAMPLER+=" --id bsampler1"
|
||||
|
Reference in New Issue
Block a user