From 1fa5e66dbc1aac73221e5c4c9f9be899921706aa Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Tue, 11 Sep 2018 10:27:31 -0600 Subject: [PATCH 21/52] btl/vader: ensure that the send tag is always written last To ensure fast box entries are complete when processed by the receiving process the tag must be written last. This includes a zero header for the next fast box entry (in some cases). This commit fixes two instances where the tag was written too early. In one case, on 32-bit systems it is possible for the tag part of the header to be written before the size. The second instance is an ordering issue. The zero header was being written after the fastbox header. Fixes #5375, #5638 Signed-off-by: Nathan Hjelm (cherry picked from commit 850fbff441756b2f9cde1007ead3e37ce22599c2) Signed-off-by: Nathan Hjelm --- opal/mca/btl/vader/btl_vader_fbox.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/opal/mca/btl/vader/btl_vader_fbox.h b/opal/mca/btl/vader/btl_vader_fbox.h index 7c0cdd5..df1fb92 100644 --- a/opal/mca/btl/vader/btl_vader_fbox.h +++ b/opal/mca/btl/vader/btl_vader_fbox.h @@ -50,9 +50,10 @@ void mca_btl_vader_poll_handle_frag (mca_btl_vader_hdr_t *hdr, mca_btl_base_endp static inline void mca_btl_vader_fbox_set_header (mca_btl_vader_fbox_hdr_t *hdr, uint16_t tag, uint16_t seq, uint32_t size) { - mca_btl_vader_fbox_hdr_t tmp = {.data = {.tag = tag, .seq = seq, .size = size}}; - opal_atomic_wmb (); + mca_btl_vader_fbox_hdr_t tmp = {.data = {.tag = 0, .seq = seq, .size = size}}; hdr->ival = tmp.ival; + opal_atomic_wmb (); + hdr->data.tag = tag; } /* attempt to reserve a contiguous segment from the remote ep */ @@ -138,9 +139,6 @@ static inline bool mca_btl_vader_fbox_sendi (mca_btl_base_endpoint_t *ep, unsign memcpy (data + header_size, payload, payload_size); } - /* write out part of the header now. the tag will be written when the data is available */ - mca_btl_vader_fbox_set_header (MCA_BTL_VADER_FBOX_HDR(dst), tag, ep->fbox_out.seq++, data_size); - end += size; if (OPAL_UNLIKELY(fbox_size == end)) { @@ -152,6 +150,9 @@ static inline bool mca_btl_vader_fbox_sendi (mca_btl_base_endpoint_t *ep, unsign MCA_BTL_VADER_FBOX_HDR(ep->fbox_out.buffer + end)->ival = 0; } + /* write out part of the header now. the tag will be written when the data is available */ + mca_btl_vader_fbox_set_header (MCA_BTL_VADER_FBOX_HDR(dst), tag, ep->fbox_out.seq++, data_size); + /* align the buffer */ ep->fbox_out.end = ((uint32_t) hbs << 31) | end; opal_atomic_wmb (); -- 1.8.3.1