diff --git a/.gitignore b/.gitignore index 5ce21a5..12085a8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -SOURCES/libfabric-1.5.3.tar.gz +SOURCES/libfabric-1.6.1.tar.gz diff --git a/.libfabric.metadata b/.libfabric.metadata index 7d36d88..0f2ffa3 100644 --- a/.libfabric.metadata +++ b/.libfabric.metadata @@ -1 +1 @@ -8d94f41e7b5b33e18865d30cd05ebb668b7a4e44 SOURCES/libfabric-1.5.3.tar.gz +f5933c41c05473aff1050db6cc6a5bd895202c0f SOURCES/libfabric-1.6.1.tar.gz diff --git a/SOURCES/0001-Revert-prov-psm2-Avoid-long-delay-in-psm2_ep_close.patch b/SOURCES/0001-Revert-prov-psm2-Avoid-long-delay-in-psm2_ep_close.patch new file mode 100644 index 0000000..ba9867d --- /dev/null +++ b/SOURCES/0001-Revert-prov-psm2-Avoid-long-delay-in-psm2_ep_close.patch @@ -0,0 +1,195 @@ +From dafc07e38c7c2af1dec371276dec08da39e1636a Mon Sep 17 00:00:00 2001 +From: Jianxin Xiong +Date: Wed, 12 Sep 2018 08:53:32 -0700 +Subject: [PATCH] Revert "prov/psm2: Avoid long delay in psm2_ep_close" + +This reverts commit 7741df0db37085c1a49c05185c9b3f8170981661. + +Sporadic assertion failures have been observed inside psm2_ep_disconnect2(). +Disable the patch until the issue is fixed. + +Signed-off-by: Jianxin Xiong +--- + prov/psm2/src/psmx2_av.c | 79 ++++++++++++++++++------------------------ + prov/psm2/src/psmx2_trx_ctxt.c | 32 +++-------------- + 2 files changed, 39 insertions(+), 72 deletions(-) + +diff --git a/prov/psm2/src/psmx2_av.c b/prov/psm2/src/psmx2_av.c +index 815407405..63c6e5beb 100644 +--- a/prov/psm2/src/psmx2_av.c ++++ b/prov/psm2/src/psmx2_av.c +@@ -32,40 +32,6 @@ + + #include "psmx2.h" + +-static void psmx2_set_epaddr_context(struct psmx2_trx_ctxt *trx_ctxt, +- psm2_epid_t epid, psm2_epaddr_t epaddr) +-{ +- struct psmx2_epaddr_context *context; +- +- context = (void *)psm2_epaddr_getctxt(epaddr); +- if (context) { +- if (context->trx_ctxt != trx_ctxt || context->epid != epid) { +- FI_WARN(&psmx2_prov, FI_LOG_AV, +- "trx_ctxt or epid doesn't match\n"); +- context = NULL; +- } +- } +- +- if (context) +- return; +- +- context = malloc(sizeof *context); +- if (!context) { +- FI_WARN(&psmx2_prov, FI_LOG_AV, +- "cannot allocate context\n"); +- return; +- } +- +- context->trx_ctxt = trx_ctxt; +- context->epid = epid; +- context->epaddr = epaddr; +- psm2_epaddr_setctxt(epaddr, context); +- +- psmx2_lock(&trx_ctxt->peer_lock, 2); +- dlist_insert_before(&context->entry, &trx_ctxt->peer_list); +- psmx2_unlock(&trx_ctxt->peer_lock, 2); +-} +- + /* + * SEP address query protocol: + * +@@ -118,8 +84,6 @@ int psmx2_am_sep_handler(psm2_am_token_t token, psm2_amarg_t *args, + struct psmx2_fid_sep *sep; + struct psmx2_sep_query *req; + struct psmx2_fid_av *av; +- psm2_epaddr_t src_epaddr; +- psm2_epid_t src_epid; + psm2_epid_t *epids; + psm2_epid_t *buf = NULL; + int buflen; +@@ -129,15 +93,6 @@ int psmx2_am_sep_handler(psm2_am_token_t token, psm2_amarg_t *args, + cmd = PSMX2_AM_GET_OP(args[0].u32w0); + domain = trx_ctxt->domain; + +- /* +- * the implicit connection to the AM source needs also to be disconnected +- * to avoid long delay inside psm2_ep_close. make sure the source is added +- * to the peer list. +- */ +- psm2_am_get_source(token, &src_epaddr); +- psm2_epaddr_to_epid(src_epaddr, &src_epid); +- psmx2_set_epaddr_context(trx_ctxt, src_epid, src_epaddr); +- + switch (cmd) { + case PSMX2_AM_REQ_SEP_QUERY: + sep_id = args[0].u32w1; +@@ -226,6 +181,40 @@ static inline double psmx2_conn_timeout(int sec) + return sec * 1e9; + } + ++static void psmx2_set_epaddr_context(struct psmx2_trx_ctxt *trx_ctxt, ++ psm2_epid_t epid, psm2_epaddr_t epaddr) ++{ ++ struct psmx2_epaddr_context *context; ++ ++ context = (void *)psm2_epaddr_getctxt(epaddr); ++ if (context) { ++ if (context->trx_ctxt != trx_ctxt || context->epid != epid) { ++ FI_WARN(&psmx2_prov, FI_LOG_AV, ++ "trx_ctxt or epid doesn't match\n"); ++ context = NULL; ++ } ++ } ++ ++ if (context) ++ return; ++ ++ context = malloc(sizeof *context); ++ if (!context) { ++ FI_WARN(&psmx2_prov, FI_LOG_AV, ++ "cannot allocate context\n"); ++ return; ++ } ++ ++ context->trx_ctxt = trx_ctxt; ++ context->epid = epid; ++ context->epaddr = epaddr; ++ psm2_epaddr_setctxt(epaddr, context); ++ ++ psmx2_lock(&trx_ctxt->peer_lock, 2); ++ dlist_insert_before(&context->entry, &trx_ctxt->peer_list); ++ psmx2_unlock(&trx_ctxt->peer_lock, 2); ++} ++ + int psmx2_epid_to_epaddr(struct psmx2_trx_ctxt *trx_ctxt, + psm2_epid_t epid, psm2_epaddr_t *epaddr) + { +diff --git a/prov/psm2/src/psmx2_trx_ctxt.c b/prov/psm2/src/psmx2_trx_ctxt.c +index 6dd3196e5..709ced94f 100644 +--- a/prov/psm2/src/psmx2_trx_ctxt.c ++++ b/prov/psm2/src/psmx2_trx_ctxt.c +@@ -124,10 +124,6 @@ void psmx2_trx_ctxt_disconnect_peers(struct psmx2_trx_ctxt *trx_ctxt) + struct psmx2_epaddr_context *peer; + struct dlist_entry peer_list; + psm2_amarg_t arg; +- psm2_epaddr_t *epaddrs; +- psm2_error_t *errors; +- int peer_count = 0; +- int i = 0; + + arg.u32w0 = PSMX2_AM_REQ_TRX_CTXT_DISCONNECT; + +@@ -137,36 +133,17 @@ void psmx2_trx_ctxt_disconnect_peers(struct psmx2_trx_ctxt *trx_ctxt) + dlist_foreach_safe(&trx_ctxt->peer_list, item, tmp) { + dlist_remove(item); + dlist_insert_before(item, &peer_list); +- peer_count++; + } + psmx2_unlock(&trx_ctxt->peer_lock, 2); + +- if (!peer_count) +- return; +- +- epaddrs = malloc(peer_count * sizeof(*epaddrs)); +- errors = malloc(peer_count * sizeof(*errors)); +- + dlist_foreach_safe(&peer_list, item, tmp) { + peer = container_of(item, struct psmx2_epaddr_context, entry); +- if (epaddrs) +- epaddrs[i++] = peer->epaddr; +- if (psmx2_env.disconnect) { +- FI_INFO(&psmx2_prov, FI_LOG_CORE, "epaddr: %p\n", peer->epaddr); +- psm2_am_request_short(peer->epaddr, PSMX2_AM_TRX_CTXT_HANDLER, +- &arg, 1, NULL, 0, 0, NULL, NULL); +- } ++ FI_INFO(&psmx2_prov, FI_LOG_CORE, "epaddr: %p\n", peer->epaddr); ++ psm2_am_request_short(peer->epaddr, PSMX2_AM_TRX_CTXT_HANDLER, ++ &arg, 1, NULL, 0, 0, NULL, NULL); + psm2_epaddr_setctxt(peer->epaddr, NULL); + free(peer); + } +- +- /* disconnect locally to avoid long delay inside psm2_ep_close() */ +- if (epaddrs && errors) +- psm2_ep_disconnect2(trx_ctxt->psm2_ep, peer_count, epaddrs, NULL, +- errors, PSM2_EP_DISCONNECT_FORCE, 0); +- +- free(errors); +- free(epaddrs); + } + + static const char *psmx2_usage_flags_to_string(int usage_flags) +@@ -203,7 +180,8 @@ void psmx2_trx_ctxt_free(struct psmx2_trx_ctxt *trx_ctxt, int usage_flags) + dlist_remove(&trx_ctxt->entry); + psmx2_unlock(&trx_ctxt->domain->trx_ctxt_lock, 1); + +- psmx2_trx_ctxt_disconnect_peers(trx_ctxt); ++ if (psmx2_env.disconnect) ++ psmx2_trx_ctxt_disconnect_peers(trx_ctxt); + + if (trx_ctxt->am_initialized) + psmx2_am_fini(trx_ctxt); +-- +2.14.4 + diff --git a/SOURCES/0001-prov-verbs-Add-support-of-different-CQ-formats-for-t.patch b/SOURCES/0001-prov-verbs-Add-support-of-different-CQ-formats-for-t.patch deleted file mode 100644 index 4620283..0000000 --- a/SOURCES/0001-prov-verbs-Add-support-of-different-CQ-formats-for-t.patch +++ /dev/null @@ -1,184 +0,0 @@ -From d5b9233daf158d2402f6dbc81ea80b07d0fe9003 Mon Sep 17 00:00:00 2001 -From: Dmitry Gladkov -Date: Fri, 22 Dec 2017 17:20:20 +0300 -Subject: [PATCH] prov/verbs: Add support of different CQ formats for the - verbs/RDM - -Signed-off-by: Dmitry Gladkov ---- - prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c | 70 +++++++++++++++++++++++++++------ - prov/verbs/src/ep_rdm/verbs_rdm.h | 1 - - prov/verbs/src/fi_verbs.h | 23 ++++++----- - 3 files changed, 73 insertions(+), 21 deletions(-) - -diff --git a/prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c b/prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c -index 5b4065fb..3612851b 100644 ---- a/prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c -+++ b/prov/verbs/src/ep_rdm/verbs_cq_ep_rdm.c -@@ -51,7 +51,6 @@ static ssize_t fi_ibv_rdm_tagged_cq_readfrom(struct fid_cq *cq, void *buf, - { - struct fi_ibv_rdm_cq *_cq = - container_of(cq, struct fi_ibv_rdm_cq, cq_fid); -- struct fi_cq_tagged_entry *entry = buf; - struct fi_ibv_rdm_request *cq_entry; - size_t ret = 0; - -@@ -66,13 +65,7 @@ static ssize_t fi_ibv_rdm_tagged_cq_readfrom(struct fid_cq *cq, void *buf, - - src_addr[ret] = - _cq->ep->av->conn_to_addr(_cq->ep, cq_entry->minfo.conn); -- entry[ret].op_context = cq_entry->context; -- entry[ret].flags = (cq_entry->comp_flags & ~FI_COMPLETION); -- entry[ret].len = cq_entry->len; -- entry[ret].buf = (cq_entry->comp_flags & FI_TRANSMIT) ? -- cq_entry->src_addr : cq_entry->dest_buf; -- entry[ret].data = cq_entry->imm; -- entry[ret].tag = cq_entry->minfo.tag; -+ _cq->read_entry(cq_entry, ret, buf); - - if (cq_entry->state.eager == FI_IBV_STATE_EAGER_READY_TO_FREE) { - FI_IBV_RDM_DBG_REQUEST("to_pool: ", cq_entry, -@@ -116,7 +109,6 @@ ssize_t fi_ibv_rdm_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count, - ((timeout < 0) ? SIZE_MAX : (fi_gettime_ms() + timeout)); - size_t counter = 0; - ssize_t ret = 0; -- struct fi_cq_tagged_entry *cqe_buf = buf; - struct fi_ibv_rdm_cq *_cq = container_of(cq, struct fi_ibv_rdm_cq, - cq_fid); - switch (_cq->wait_cond) { -@@ -130,7 +122,9 @@ ssize_t fi_ibv_rdm_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count, - } - - do { -- ret = fi_ibv_rdm_tagged_cq_readfrom(cq, &cqe_buf[counter], -+ ret = fi_ibv_rdm_tagged_cq_readfrom(cq, -+ ((char *)buf + -+ (counter * _cq->entry_size)), - threshold - counter, - src_addr); - counter += (ret > 0) ? ret : 0; -@@ -260,6 +254,50 @@ static struct fi_ops fi_ibv_rdm_cq_fi_ops = { - .ops_open = fi_no_ops_open, - }; - -+static void fi_ibv_rdm_cq_read_context_entry(struct fi_ibv_rdm_request *cq_entry, -+ int i, void *buf) -+{ -+ struct fi_cq_entry *entry = buf; -+ -+ entry[i].op_context = cq_entry->context; -+ } -+ -+ static void fi_ibv_rdm_cq_read_msg_entry(struct fi_ibv_rdm_request *cq_entry, -+ int i, void *buf) -+ { -+ struct fi_cq_msg_entry *entry = buf; -+ -+ entry[i].op_context = cq_entry->context; -+ entry[i].flags = (cq_entry->comp_flags & ~FI_COMPLETION); -+ entry[i].len = cq_entry->len; -+ } -+ -+ static void fi_ibv_rdm_cq_read_data_entry(struct fi_ibv_rdm_request *cq_entry, -+ int i, void *buf) -+ { -+ struct fi_cq_data_entry *entry = buf; -+ -+ entry[i].op_context = cq_entry->context; -+ entry[i].flags = (cq_entry->comp_flags & ~FI_COMPLETION); -+ entry[i].len = cq_entry->len; -+ entry[i].buf = (cq_entry->comp_flags & FI_TRANSMIT) ? -+ cq_entry->src_addr : cq_entry->dest_buf; -+ entry[i].data = cq_entry->imm; -+ } -+ -+ static void fi_ibv_rdm_cq_read_tagged_entry(struct fi_ibv_rdm_request *cq_entry, -+ int i, void *buf) -+ { -+ struct fi_cq_tagged_entry *entry = buf; -+ -+ entry[i].op_context = cq_entry->context; -+ entry[i].flags = (cq_entry->comp_flags & ~FI_COMPLETION); -+ entry[i].len = cq_entry->len; -+ entry[i].buf = (cq_entry->comp_flags & FI_TRANSMIT) ? -+ cq_entry->src_addr : cq_entry->dest_buf; -+ entry[i].data = cq_entry->imm; -+ entry[i].tag = cq_entry->minfo.tag; -+ } - - int fi_ibv_rdm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, - struct fid_cq **cq, void *context) -@@ -296,12 +334,22 @@ int fi_ibv_rdm_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, - _cq->cq_fid.ops = &fi_ibv_rdm_cq_ops; - - switch (attr->format) { -- case FI_CQ_FORMAT_UNSPEC: - case FI_CQ_FORMAT_CONTEXT: -+ _cq->entry_size = sizeof(struct fi_cq_entry); -+ _cq->read_entry = fi_ibv_rdm_cq_read_context_entry; -+ break; - case FI_CQ_FORMAT_MSG: -+ _cq->entry_size = sizeof(struct fi_cq_msg_entry); -+ _cq->read_entry = fi_ibv_rdm_cq_read_msg_entry; -+ break; - case FI_CQ_FORMAT_DATA: -+ _cq->entry_size = sizeof(struct fi_cq_data_entry); -+ _cq->read_entry = fi_ibv_rdm_cq_read_data_entry; -+ break; -+ case FI_CQ_FORMAT_UNSPEC: - case FI_CQ_FORMAT_TAGGED: - _cq->entry_size = sizeof(struct fi_cq_tagged_entry); -+ _cq->read_entry = fi_ibv_rdm_cq_read_tagged_entry; - break; - default: - ret = -FI_ENOSYS; -diff --git a/prov/verbs/src/ep_rdm/verbs_rdm.h b/prov/verbs/src/ep_rdm/verbs_rdm.h -index 6230c975..71aabea3 100644 ---- a/prov/verbs/src/ep_rdm/verbs_rdm.h -+++ b/prov/verbs/src/ep_rdm/verbs_rdm.h -@@ -309,7 +309,6 @@ struct fi_ibv_rdm_ep { - int use_odp; - int scq_depth; - int rcq_depth; -- int cqread_bunch_size; - - int is_closing; - int recv_preposted_threshold; -diff --git a/prov/verbs/src/fi_verbs.h b/prov/verbs/src/fi_verbs.h -index 43f11255..06becb78 100644 ---- a/prov/verbs/src/fi_verbs.h -+++ b/prov/verbs/src/fi_verbs.h -@@ -257,16 +257,21 @@ struct fi_ibv_cq { - struct util_buf_pool *wce_pool; - }; - -+struct fi_ibv_rdm_request; -+typedef void (*fi_ibv_rdm_cq_read_entry)(struct fi_ibv_rdm_request *cq_entry, -+ int index, void *buf); -+ - struct fi_ibv_rdm_cq { -- struct fid_cq cq_fid; -- struct fi_ibv_domain *domain; -- struct fi_ibv_rdm_ep *ep; -- struct dlist_entry request_cq; -- struct dlist_entry request_errcq; -- uint64_t flags; -- size_t entry_size; -- int read_bunch_size; -- enum fi_cq_wait_cond wait_cond; -+ struct fid_cq cq_fid; -+ struct fi_ibv_domain *domain; -+ struct fi_ibv_rdm_ep *ep; -+ struct dlist_entry request_cq; -+ struct dlist_entry request_errcq; -+ uint64_t flags; -+ size_t entry_size; -+ fi_ibv_rdm_cq_read_entry read_entry; -+ int read_bunch_size; -+ enum fi_cq_wait_cond wait_cond; - }; - - int fi_ibv_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, --- -2.15.GIT - diff --git a/SPECS/libfabric.spec b/SPECS/libfabric.spec index 714f5b1..e5ff731 100644 --- a/SPECS/libfabric.spec +++ b/SPECS/libfabric.spec @@ -1,12 +1,12 @@ Name: libfabric -Version: 1.5.3 -Release: 1%{?dist} +Version: 1.6.1 +Release: 2%{?dist} Summary: User-space RDMA Fabric Interfaces Group: System Environment/Libraries License: GPLv2 or BSD Url: http://www.github.com/ofiwg/libfabric Source: %{name}-%{version}.tar.gz -Patch2: 0001-prov-verbs-Add-support-of-different-CQ-formats-for-t.patch +Patch1: 0001-Revert-prov-psm2-Avoid-long-delay-in-psm2_ep_close.patch BuildRequires: librdmacm-devel BuildRequires: libibverbs-devel >= 1.2.0 @@ -42,7 +42,7 @@ Development files for the libfabric library. %prep %setup -q -%patch2 -p1 +%patch1 -p1 %build @@ -79,6 +79,14 @@ rm -f %{buildroot}%{_libdir}/*.la %{_mandir}/man7/* %changelog +* Sat Sep 22 2018 Honggang Li - 1.6.1-2 +- Revert a psm2 commit to avoid sporadic assertion failures +- Resolves: bz1631874 + +* Tue Jun 5 2018 Honggang Li - 1.6.1-1 +- Rebase to latest release 1.6.1 +- Resolves: bz1483568 + * Wed Jan 10 2018 Honggang Li - 1.5.3-1 - Rebase to latest release 1.5.3 - Resolves: bz1533293