/**
* Copyright (C) Mellanox Technologies Ltd. 2001-2014.  ALL RIGHTS RESERVED.
* Copyright (C) UT-Battelle, LLC. 2014. ALL RIGHTS RESERVED.
* See file LICENSE for terms.
*/

#ifndef UCT_IB_VERBS_H
#define UCT_IB_VERBS_H

#ifdef HAVE_CONFIG_H
#  include "config.h"
#endif

#include <infiniband/verbs.h>
#ifdef HAVE_VERBS_EXP_H
#include <infiniband/verbs_exp.h>
#endif

#include <errno.h>

#ifndef HAVE_VERBS_EXP_H
#  define IBV_EXP_SEND_INLINE              IBV_SEND_INLINE
#  define IBV_EXP_SEND_SIGNALED            IBV_SEND_SIGNALED
#  define IBV_EXP_SEND_SOLICITED           IBV_SEND_SOLICITED
#  define IBV_EXP_SEND_FENCE               IBV_SEND_FENCE
#  define IBV_EXP_QP_STATE                 IBV_QP_STATE
#  define IBV_EXP_QP_PKEY_INDEX            IBV_QP_PKEY_INDEX
#  define IBV_EXP_QP_PORT                  IBV_QP_PORT
#  define IBV_EXP_QP_PATH_MTU              IBV_QP_PATH_MTU
#  define IBV_EXP_QP_TIMEOUT               IBV_QP_TIMEOUT
#  define IBV_EXP_QP_AV                    IBV_QP_AV
#  define IBV_EXP_QP_RETRY_CNT             IBV_QP_RETRY_CNT
#  define IBV_EXP_QP_RNR_RETRY             IBV_QP_RNR_RETRY
#  define IBV_EXP_QP_MAX_QP_RD_ATOMIC      IBV_QP_MAX_QP_RD_ATOMIC
#  define IBV_EXP_ACCESS_REMOTE_WRITE      IBV_ACCESS_REMOTE_WRITE
#  define IBV_EXP_ACCESS_REMOTE_READ       IBV_ACCESS_REMOTE_READ
#  define IBV_EXP_ACCESS_REMOTE_ATOMIC     IBV_ACCESS_REMOTE_ATOMIC
#  define ibv_exp_reg_shared_mr            ibv_reg_shared_mr_ex
#  define ibv_exp_reg_shared_mr_in         ibv_reg_shared_mr_in
#  define ibv_exp_query_device             ibv_query_device
#  define ibv_exp_device_attr              ibv_device_attr
#  define ibv_exp_send_wr                  ibv_send_wr
#  define exp_opcode                       opcode
#  define ibv_exp_post_send                ibv_post_send
#  define IBV_EXP_WR_NOP                   IBV_WR_NOP
#  define IBV_EXP_WR_SEND                  IBV_WR_SEND
#  define IBV_EXP_WR_RDMA_WRITE            IBV_WR_RDMA_WRITE
#  define IBV_EXP_WR_RDMA_READ             IBV_WR_RDMA_READ
#  define IBV_EXP_WR_ATOMIC_FETCH_AND_ADD  IBV_WR_ATOMIC_FETCH_AND_ADD
#  define IBV_EXP_WR_ATOMIC_CMP_AND_SWP    IBV_WR_ATOMIC_CMP_AND_SWP
#  define ibv_exp_qp_init_attr             ibv_qp_init_attr
#  define ibv_exp_port_attr                ibv_port_attr
#  define ibv_exp_query_port               ibv_query_port
#  define exp_device_cap_flags             device_cap_flags
#  define ibv_exp_create_qp                ibv_create_qp

#  define IBV_SHARED_MR_ACCESS_FLAGS(_shared_mr)    ((_shared_mr)->exp_access)
#  define IBV_EXP_DEVICE_ATTR_SET_COMP_MASK(_attr)
#  define IBV_EXP_PORT_ATTR_SET_COMP_MASK(_attr)

#else
#  define IBV_SHARED_MR_ACCESS_FLAGS(_shared_mr)    ((_shared_mr)->access)
#if HAVE_DECL_IBV_EXP_DEVICE_ATTR_RESERVED_2
#  define IBV_EXP_DEVICE_ATTR_SET_COMP_MASK(_attr)  do { \
            (_attr)->comp_mask = 0xffffffff; \
            (_attr)->comp_mask_2 = (IBV_EXP_DEVICE_ATTR_RESERVED_2 - 1); \
        } while (0)
#else
#  define IBV_EXP_DEVICE_ATTR_SET_COMP_MASK(_attr)  (_attr)->comp_mask = (IBV_EXP_DEVICE_ATTR_RESERVED - 1)
#endif /* HAVE_DECL_IBV_EXP_DEVICE_ATTR_RESERVED_2 */
#  define IBV_EXP_PORT_ATTR_SET_COMP_MASK(_attr)    (_attr)->comp_mask = 0
#endif /* HAVE_VERBS_EXP_H */


/*
 * Contiguous pages support
 */
#if HAVE_DECL_IBV_EXP_DEVICE_MR_ALLOCATE
#  define IBV_EXP_HAVE_CONTIG_PAGES(_attr)          ((_attr)->exp_device_cap_flags & IBV_EXP_DEVICE_MR_ALLOCATE)
#else
#  define IBV_EXP_HAVE_CONTIG_PAGES(_attr)         0
#endif


/*
 * On-demand paging support
 */
#if HAVE_STRUCT_IBV_EXP_DEVICE_ATTR_ODP_CAPS
#  define IBV_EXP_HAVE_ODP(_attr)                   ((_attr)->odp_caps.general_odp_caps & IBV_EXP_ODP_SUPPORT)
#  define IBV_EXP_ODP_CAPS(_attr, _xport)           ((_attr)->odp_caps.per_transport_caps._xport##_odp_caps)
#else
#  define IBV_EXP_HAVE_ODP(_attr)                   0
#  define IBV_EXP_ODP_CAPS(_attr, _xport)           0
#endif

#if !HAVE_DECL_IBV_EXP_ACCESS_ON_DEMAND
#  define IBV_EXP_ACCESS_ON_DEMAND                  0
#endif

#if !HAVE_DECL_IBV_EXP_PREFETCH_WRITE_ACCESS
#  define IBV_EXP_PREFETCH_WRITE_ACCESS             IBV_EXP_ACCESS_LOCAL_WRITE
#endif

/*
 * DC support
 */
#if HAVE_DECL_IBV_EXP_DEVICE_DC_TRANSPORT && HAVE_STRUCT_IBV_EXP_DEVICE_ATTR_EXP_DEVICE_CAP_FLAGS
#  define IBV_DEVICE_HAS_DC(_attr)                  ((_attr)->exp_device_cap_flags & IBV_EXP_DEVICE_DC_TRANSPORT)
#else
#  define IBV_DEVICE_HAS_DC(_attr)                  0
#endif /* HAVE_DECL_IBV_EXP_DEVICE_DC_TRANSPORT */


/*
 * NOP support
 */
#if HAVE_DECL_IBV_EXP_WR_NOP
#  define IBV_DEVICE_HAS_NOP(_attr)                 ((_attr)->exp_device_cap_flags & IBV_EXP_DEVICE_NOP)
#else
#  define IBV_DEVICE_HAS_NOP(_attr)                 0
#endif /* HAVE_DECL_IBV_EXP_WR_NOP */

/*
 * Adaptive Routing support
 */
#if HAVE_DECL_IBV_EXP_QP_OOO_RW_DATA_PLACEMENT
#  define UCX_IB_DEV_IS_OOO_SUPPORTED(_attr, _transport)  \
    (((_attr)->comp_mask & IBV_EXP_DEVICE_ATTR_OOO_CAPS) && \
     ((_attr)->ooo_caps._transport##_caps & IBV_EXP_OOO_SUPPORT_RW_DATA_PLACEMENT))
#else
#  define UCX_IB_DEV_IS_OOO_SUPPORTED(_ibdev, _transport)   0
#endif


/*
 * Safe setenv
 */
#if HAVE_DECL_IBV_EXP_SETENV
#  define ibv_exp_unsetenv(_c, _n)                  0
#else
#  define ibv_exp_setenv(_c, _n, _v, _o)            setenv(_n, _v, _o)
#  define ibv_exp_unsetenv(_c, _n)                  unsetenv(_n)
#endif


/*
 * CQ overrun support
 */
#if HAVE_DECL_IBV_EXP_CQ_IGNORE_OVERRUN
static inline int ibv_exp_cq_ignore_overrun(struct ibv_cq *cq)
{
    struct ibv_exp_cq_attr cq_attr = {0};
    cq_attr.comp_mask    = IBV_EXP_CQ_ATTR_CQ_CAP_FLAGS;
    cq_attr.cq_cap_flags = IBV_EXP_CQ_IGNORE_OVERRUN;
    return ibv_exp_modify_cq(cq, &cq_attr, IBV_EXP_CQ_CAP_FLAGS);
}
#else
static inline int ibv_exp_cq_ignore_overrun(struct ibv_cq *cq)
{
    errno = ENOSYS;
    return -1;
}
#endif /* HAVE_IBV_EXP_CQ_IGNORE_OVERRUN */


/*
 * Atomics support
 */
#if HAVE_DECL_IBV_EXP_ATOMIC_HCA_REPLY_BE
#  define IBV_EXP_HAVE_ATOMIC_HCA(_attr)            ((_attr)->exp_atomic_cap == IBV_EXP_ATOMIC_HCA)
#  define IBV_EXP_HAVE_ATOMIC_GLOB(_attr)           ((_attr)->exp_atomic_cap == IBV_EXP_ATOMIC_GLOB)
#  define IBV_EXP_HAVE_ATOMIC_HCA_REPLY_BE(_attr)   ((_attr)->exp_atomic_cap == IBV_EXP_ATOMIC_HCA_REPLY_BE)
#else
#  define IBV_EXP_HAVE_ATOMIC_HCA(_attr)            ((_attr)->atomic_cap == IBV_ATOMIC_HCA)
#  define IBV_EXP_HAVE_ATOMIC_GLOB(_attr)           ((_attr)->atomic_cap == IBV_ATOMIC_GLOB)
#  define IBV_EXP_HAVE_ATOMIC_HCA_REPLY_BE(_attr)   0
#endif /* HAVE_DECL_IBV_EXP_ATOMIC_HCA_REPLY_BE */


/* Ethernet link layer */
#if HAVE_DECL_IBV_LINK_LAYER_ETHERNET
#  define IBV_PORT_IS_LINK_LAYER_ETHERNET(_attr)    ((_attr)->link_layer == IBV_LINK_LAYER_ETHERNET)
#else
#  define IBV_PORT_IS_LINK_LAYER_ETHERNET(_attr)    0
#endif


/*
 * HW tag matching
 */
#if IBV_EXP_HW_TM
   /* TM (eager) is supported if tm_caps.max_num_tags is not 0. */
#  define IBV_DEVICE_TM_CAPS(_dev, _field)  ((_dev)->dev_attr.tm_caps._field)

   /* If the gap between SW and HW counters is more than 32K, all messages will
    * be dropped and RNR ACK will be returned. Set threshold to 16K to avoid
    * hitting this gap at all. */
#  define IBV_DEVICE_MAX_UNEXP_COUNT        UCS_BIT(14)

#  define IBV_DEVICE_MIN_UWQ_POST           33
#else
#  define IBV_DEVICE_TM_CAPS(_dev, _field)  0
#  define IBV_EXP_TM_CAP_RC                 0
#endif

#ifndef IBV_EXP_HW_TM_DC
#  define IBV_EXP_TM_CAP_DC                 0
#endif

#if !HAVE_DECL_IBV_EXP_CREATE_SRQ
#  define ibv_exp_create_srq_attr           ibv_srq_init_attr
#endif

typedef uint8_t uct_ib_uint24_t[3];

static inline void uct_ib_pack_uint24(uct_ib_uint24_t buf, const uint32_t qp_num)
{

    buf[0] = (qp_num >> 0)  & 0xFF;
    buf[1] = (qp_num >> 8)  & 0xFF;
    buf[2] = (qp_num >> 16) & 0xFF;
}

static inline uint32_t uct_ib_unpack_uint24(const uct_ib_uint24_t buf)
{
    return buf[0] | ((uint32_t)buf[1] << 8) | ((uint32_t)buf[2] << 16);
}

typedef struct uct_ib_qpnum {
    uct_ib_uint24_t qp_num;
} uct_ib_qpnum_t;

#endif /* UCT_IB_VERBS_H */
