From 5d658937ce874422c66ac4c7d9723df8bb69ecd7 Mon Sep 17 00:00:00 2001
From: Nathan Hjelm <hjelmn@me.com>
Date: Fri, 5 Oct 2018 16:30:06 -0600
Subject: [PATCH 40/52] btl/vader: fix race condition in writing header

Signed-off-by: Nathan Hjelm <hjelmn@me.com>
(cherry picked from commit 8291f6722d890efd15333bf7b26f0d07952fa41e)
Signed-off-by: Nathan Hjelm <hjelmn@me.com>
---
 opal/mca/btl/vader/btl_vader_fbox.h | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/opal/mca/btl/vader/btl_vader_fbox.h b/opal/mca/btl/vader/btl_vader_fbox.h
index 6df9a2d..4ebfde8 100644
--- a/opal/mca/btl/vader/btl_vader_fbox.h
+++ b/opal/mca/btl/vader/btl_vader_fbox.h
@@ -29,6 +29,10 @@ typedef union mca_btl_vader_fbox_hdr_t {
         /** sequence number */
         uint16_t  seq;
     } data;
+    struct {
+        uint32_t value0;
+        uint32_t value1;
+    } data_i32;
     uint64_t ival;
 } mca_btl_vader_fbox_hdr_t;
 
@@ -50,19 +54,20 @@ void mca_btl_vader_poll_handle_frag (mca_btl_vader_hdr_t *hdr, mca_btl_base_endp
 static inline void mca_btl_vader_fbox_set_header (mca_btl_vader_fbox_hdr_t *hdr, uint16_t tag,
                                                   uint16_t seq, uint32_t size)
 {
-    mca_btl_vader_fbox_hdr_t tmp = {.data = {.tag = 0, .seq = seq, .size = size}};
-    hdr->ival = tmp.ival;
+    mca_btl_vader_fbox_hdr_t tmp = {.data = {.tag = tag, .seq = seq, .size = size}};
+    /* clear out existing tag/seq */
+    hdr->data_i32.value1 = 0;
+    opal_atomic_wmb ();
+    hdr->data_i32.value0 = size;
     opal_atomic_wmb ();
-    hdr->data.tag = tag;
+    hdr->data_i32.value1 = tmp.data_i32.value1;
 }
 
 static inline mca_btl_vader_fbox_hdr_t mca_btl_vader_fbox_read_header (mca_btl_vader_fbox_hdr_t *hdr)
 {
-    mca_btl_vader_fbox_hdr_t tmp;
-    uint16_t tag = hdr->data.tag;
+    mca_btl_vader_fbox_hdr_t tmp = {.data_i32 = {.value1 = hdr->data_i32.value1}};;
     opal_atomic_rmb ();
-    tmp.ival = hdr->ival;
-    tmp.data.tag = tag;
+    tmp.data_i32.value0 = hdr->data_i32.value0;
     return tmp;
 }
 
-- 
1.8.3.1