From 3785c0e369177a479e383b4ad5eba61fca89db36 Mon Sep 17 00:00:00 2001
From: Dennis Klein <d.klein@gsi.de>
Date: Thu, 15 Feb 2018 18:23:46 +0100
Subject: [PATCH] FairMQ: Implement ofi::TransportFactory ctor/dtor

---
 fairmq/FairMQTransportFactory.h |  2 ++
 fairmq/ofi/TransportFactory.cxx | 58 ++++++++++++++++++++++++++++++++-
 fairmq/ofi/TransportFactory.h   |  3 ++
 3 files changed, 62 insertions(+), 1 deletion(-)

diff --git a/fairmq/FairMQTransportFactory.h b/fairmq/FairMQTransportFactory.h
index 309fa65d..dde24a21 100644
--- a/fairmq/FairMQTransportFactory.h
+++ b/fairmq/FairMQTransportFactory.h
@@ -73,6 +73,8 @@ class FairMQTransportFactory
 
     virtual ~FairMQTransportFactory() {};
 
+    struct TransportFactoryError : std::runtime_error { using std::runtime_error::runtime_error; };
+
     static auto CreateTransportFactory(const std::string& type, const std::string& id = "", const FairMQProgOptions* config = nullptr) -> std::shared_ptr<FairMQTransportFactory>;
 
     static void FairMQNoCleanup(void* /*data*/, void* /*obj*/)
diff --git a/fairmq/ofi/TransportFactory.cxx b/fairmq/ofi/TransportFactory.cxx
index 8ed410d5..eea18030 100644
--- a/fairmq/ofi/TransportFactory.cxx
+++ b/fairmq/ofi/TransportFactory.cxx
@@ -7,7 +7,12 @@
  ********************************************************************************/
 
 #include <fairmq/ofi/TransportFactory.h>
+#include <fairmq/Tools.h>
+
+#include <rdma/fabric.h> // OFI libfabric
+#include <rdma/fi_errno.h>
 #include <stdexcept>
+#include <zmq.h>
 
 namespace fair
 {
@@ -19,8 +24,40 @@ namespace ofi
 using namespace std;
 
 TransportFactory::TransportFactory(const string& id, const FairMQProgOptions* config)
-: FairMQTransportFactory{id}
+    : FairMQTransportFactory{id}
+    , fZmqContext{zmq_ctx_new()}
 {
+    if (!fZmqContext)
+    {
+        throw TransportFactoryError{tools::ToString("Failed creating zmq context, reason: ", zmq_strerror(errno))};
+    }
+
+    auto ofi_hints = fi_allocinfo();
+    ofi_hints->caps = FI_MSG | FI_RMA;
+    ofi_hints->mode = FI_ASYNC_IOV;
+    ofi_hints->addr_format = FI_SOCKADDR_IN;
+    auto ofi_info = fi_allocinfo();
+    if (ofi_hints == nullptr || ofi_info == nullptr)
+    {
+        throw TransportFactoryError{"Failed allocating fi_info structs"};
+    }
+    auto res = fi_getinfo(FI_VERSION(1, 5), nullptr, nullptr, 0, ofi_hints, &ofi_info);
+    if (res != 0)
+    {
+        throw TransportFactoryError{tools::ToString("Failed querying fi_getinfo, reason: ", fi_strerror(res))};
+    }
+    for(auto cursor{ofi_info}; cursor->next != nullptr; cursor = cursor->next)
+    {
+        LOG(debug) << fi_tostr(cursor, FI_TYPE_INFO);
+    }
+    fi_freeinfo(ofi_hints);
+    fi_freeinfo(ofi_info);
+
+    int major, minor, patch;
+    zmq_version(&major, &minor, &patch);
+    auto ofi_version{fi_version()};
+    LOG(debug) << "Transport: Using ZeroMQ (" << major << "." << minor << "." << patch << ") & "
+               << "OFI libfabric (API " << FI_MAJOR(ofi_version) << "." << FI_MINOR(ofi_version) << ")";
 }
 
 auto TransportFactory::CreateMessage() const -> MessagePtr
@@ -80,6 +117,25 @@ auto TransportFactory::GetType() const -> Transport
 
 TransportFactory::~TransportFactory()
 {
+    for (int i = 0; i < 5; ++i)
+    {
+        if (zmq_ctx_term(fZmqContext) != 0)
+        {
+            if ((errno == EINTR) && (i < 4))
+            {
+                LOG(warn) << "failed closing zmq context, reason: " << zmq_strerror(errno) << ", trying again";
+            }
+            else
+            {
+                LOG(error) << "failed closing zmq context, reason: " << zmq_strerror(errno);
+                break;
+            }
+        }
+        else
+        {
+            break;
+        }
+    }
 }
 
 } /* namespace ofi */
diff --git a/fairmq/ofi/TransportFactory.h b/fairmq/ofi/TransportFactory.h
index cf82cf87..4f71fb2a 100644
--- a/fairmq/ofi/TransportFactory.h
+++ b/fairmq/ofi/TransportFactory.h
@@ -49,6 +49,9 @@ class TransportFactory : public FairMQTransportFactory
     auto GetType() const -> Transport override;
 
     ~TransportFactory() override;
+
+  private:
+    void* fZmqContext;
 }; /* class TransportFactory */  
 
 } /* namespace ofi */