// SPDX-License-Identifier: BSD-3-Clause
/* Copyright 2014-2020, Intel Corporation */

#include <string.h>
#include <xmmintrin.h>

#include "auto_flush.h"
#include "cpu.h"
#include "flush.h"
#include "memcpy_memset.h"
#include "os.h"
#include "out.h"
#include "pmem2_arch.h"
#include "valgrind_internal.h"

#define MOVNT_THRESHOLD	256

size_t Movnt_threshold = MOVNT_THRESHOLD;

/*
 * memory_barrier -- (internal) issue the fence instruction
 */
static void
memory_barrier(void)
{
	LOG(15, NULL);
	_mm_sfence();	/* ensure CLWB or CLFLUSHOPT completes */
}

/*
 * flush_clflush -- (internal) flush the CPU cache, using clflush
 */
static void
flush_clflush(const void *addr, size_t len)
{
	LOG(15, "addr %p len %zu", addr, len);

	flush_clflush_nolog(addr, len);
}

/*
 * flush_clflushopt -- (internal) flush the CPU cache, using clflushopt
 */
static void
flush_clflushopt(const void *addr, size_t len)
{
	LOG(15, "addr %p len %zu", addr, len);

	flush_clflushopt_nolog(addr, len);
}

/*
 * flush_clwb -- (internal) flush the CPU cache, using clwb
 */
static void
flush_clwb(const void *addr, size_t len)
{
	LOG(15, "addr %p len %zu", addr, len);

	flush_clwb_nolog(addr, len);
}

#if SSE2_AVAILABLE || AVX_AVAILABLE || AVX512F_AVAILABLE
#define PMEM2_F_MEM_MOVNT (PMEM2_F_MEM_WC | PMEM2_F_MEM_NONTEMPORAL)
#define PMEM2_F_MEM_MOV   (PMEM2_F_MEM_WB | PMEM2_F_MEM_TEMPORAL)

#define MEMCPY_TEMPLATE(isa, flush, perfbarrier) \
static void *\
memmove_nodrain_##isa##_##flush##perfbarrier(void *dest, const void *src, \
		size_t len, unsigned flags, flush_func flushf)\
{\
	if (len == 0 || src == dest)\
		return dest;\
\
	if (flags & PMEM2_F_MEM_NOFLUSH) \
		memmove_mov_##isa##_noflush(dest, src, len); \
	else if (flags & PMEM2_F_MEM_MOVNT)\
		memmove_movnt_##isa ##_##flush##perfbarrier(dest, src, len);\
	else if (flags & PMEM2_F_MEM_MOV)\
		memmove_mov_##isa##_##flush(dest, src, len);\
	else if (len < Movnt_threshold)\
		memmove_mov_##isa##_##flush(dest, src, len);\
	else\
		memmove_movnt_##isa##_##flush##perfbarrier(dest, src, len);\
\
	return dest;\
}

#define MEMCPY_TEMPLATE_EADR(isa, perfbarrier) \
static void *\
memmove_nodrain_##isa##_eadr##perfbarrier(void *dest, const void *src, \
		size_t len, unsigned flags, flush_func flushf)\
{\
	if (len == 0 || src == dest)\
		return dest;\
\
	if (flags & PMEM2_F_MEM_NOFLUSH)\
		memmove_mov_##isa##_noflush(dest, src, len);\
	else if (flags & PMEM2_F_MEM_NONTEMPORAL)\
		memmove_movnt_##isa##_empty##perfbarrier(dest, src, len);\
	else\
		memmove_mov_##isa##_empty(dest, src, len);\
\
	return dest;\
}

#define MEMSET_TEMPLATE(isa, flush, perfbarrier)\
static void *\
memset_nodrain_##isa##_##flush##perfbarrier(void *dest, int c, size_t len, \
		unsigned flags, flush_func flushf)\
{\
	if (len == 0)\
		return dest;\
\
	if (flags & PMEM2_F_MEM_NOFLUSH) \
		memset_mov_##isa##_noflush(dest, c, len); \
	else if (flags & PMEM2_F_MEM_MOVNT)\
		memset_movnt_##isa##_##flush##perfbarrier(dest, c, len);\
	else if (flags & PMEM2_F_MEM_MOV)\
		memset_mov_##isa##_##flush(dest, c, len);\
	else if (len < Movnt_threshold)\
		memset_mov_##isa##_##flush(dest, c, len);\
	else\
		memset_movnt_##isa##_##flush##perfbarrier(dest, c, len);\
\
	return dest;\
}

#define MEMSET_TEMPLATE_EADR(isa, perfbarrier) \
static void *\
memset_nodrain_##isa##_eadr##perfbarrier(void *dest, int c, size_t len, \
		unsigned flags, flush_func flushf)\
{\
	if (len == 0)\
		return dest;\
\
	if (flags & PMEM2_F_MEM_NOFLUSH)\
		memset_mov_##isa##_noflush(dest, c, len);\
	else if (flags & PMEM2_F_MEM_NONTEMPORAL)\
		memset_movnt_##isa##_empty##perfbarrier(dest, c, len);\
	else\
		memset_mov_##isa##_empty(dest, c, len);\
\
	return dest;\
}
#endif

#if SSE2_AVAILABLE
MEMCPY_TEMPLATE(sse2, clflush, _nobarrier)
MEMCPY_TEMPLATE(sse2, clflushopt, _nobarrier)
MEMCPY_TEMPLATE(sse2, clwb, _nobarrier)
MEMCPY_TEMPLATE_EADR(sse2, _nobarrier)

MEMSET_TEMPLATE(sse2, clflush, _nobarrier)
MEMSET_TEMPLATE(sse2, clflushopt, _nobarrier)
MEMSET_TEMPLATE(sse2, clwb, _nobarrier)
MEMSET_TEMPLATE_EADR(sse2, _nobarrier)

MEMCPY_TEMPLATE(sse2, clflush, _wcbarrier)
MEMCPY_TEMPLATE(sse2, clflushopt, _wcbarrier)
MEMCPY_TEMPLATE(sse2, clwb, _wcbarrier)
MEMCPY_TEMPLATE_EADR(sse2, _wcbarrier)

MEMSET_TEMPLATE(sse2, clflush, _wcbarrier)
MEMSET_TEMPLATE(sse2, clflushopt, _wcbarrier)
MEMSET_TEMPLATE(sse2, clwb, _wcbarrier)
MEMSET_TEMPLATE_EADR(sse2, _wcbarrier)
#endif

#if AVX_AVAILABLE
MEMCPY_TEMPLATE(avx, clflush, _nobarrier)
MEMCPY_TEMPLATE(avx, clflushopt, _nobarrier)
MEMCPY_TEMPLATE(avx, clwb, _nobarrier)
MEMCPY_TEMPLATE_EADR(avx, _nobarrier)

MEMSET_TEMPLATE(avx, clflush, _nobarrier)
MEMSET_TEMPLATE(avx, clflushopt, _nobarrier)
MEMSET_TEMPLATE(avx, clwb, _nobarrier)
MEMSET_TEMPLATE_EADR(avx, _nobarrier)

MEMCPY_TEMPLATE(avx, clflush, _wcbarrier)
MEMCPY_TEMPLATE(avx, clflushopt, _wcbarrier)
MEMCPY_TEMPLATE(avx, clwb, _wcbarrier)
MEMCPY_TEMPLATE_EADR(avx, _wcbarrier)

MEMSET_TEMPLATE(avx, clflush, _wcbarrier)
MEMSET_TEMPLATE(avx, clflushopt, _wcbarrier)
MEMSET_TEMPLATE(avx, clwb, _wcbarrier)
MEMSET_TEMPLATE_EADR(avx, _wcbarrier)
#endif

#if AVX512F_AVAILABLE
MEMCPY_TEMPLATE(avx512f, clflush, /* cstyle wa */)
MEMCPY_TEMPLATE(avx512f, clflushopt, /* */)
MEMCPY_TEMPLATE(avx512f, clwb, /* */)
MEMCPY_TEMPLATE_EADR(avx512f, /* */)

MEMSET_TEMPLATE(avx512f, clflush, /* */)
MEMSET_TEMPLATE(avx512f, clflushopt, /* */)
MEMSET_TEMPLATE(avx512f, clwb, /* */)
MEMSET_TEMPLATE_EADR(avx512f, /* */)
#endif

enum memcpy_impl {
	MEMCPY_INVALID,
	MEMCPY_SSE2,
	MEMCPY_AVX,
	MEMCPY_AVX512F
};

/*
 * use_sse2_memcpy_memset -- (internal) SSE2 detected, use it if possible
 */
static void
use_sse2_memcpy_memset(struct pmem2_arch_info *info, enum memcpy_impl *impl,
		int wc_workaround)
{
#if SSE2_AVAILABLE
	*impl = MEMCPY_SSE2;
	if (wc_workaround) {
		info->memmove_nodrain_eadr =
				memmove_nodrain_sse2_eadr_wcbarrier;
		if (info->flush == flush_clflush)
			info->memmove_nodrain =
				memmove_nodrain_sse2_clflush_wcbarrier;
		else if (info->flush == flush_clflushopt)
			info->memmove_nodrain =
				memmove_nodrain_sse2_clflushopt_wcbarrier;
		else if (info->flush == flush_clwb)
			info->memmove_nodrain =
				memmove_nodrain_sse2_clwb_wcbarrier;
		else
			ASSERT(0);

		info->memset_nodrain_eadr = memset_nodrain_sse2_eadr_wcbarrier;
		if (info->flush == flush_clflush)
			info->memset_nodrain =
				memset_nodrain_sse2_clflush_wcbarrier;
		else if (info->flush == flush_clflushopt)
			info->memset_nodrain =
				memset_nodrain_sse2_clflushopt_wcbarrier;
		else if (info->flush == flush_clwb)
			info->memset_nodrain =
				memset_nodrain_sse2_clwb_wcbarrier;
		else
			ASSERT(0);
	} else {
		info->memmove_nodrain_eadr =
				memmove_nodrain_sse2_eadr_nobarrier;
		if (info->flush == flush_clflush)
			info->memmove_nodrain =
				memmove_nodrain_sse2_clflush_nobarrier;
		else if (info->flush == flush_clflushopt)
			info->memmove_nodrain =
				memmove_nodrain_sse2_clflushopt_nobarrier;
		else if (info->flush == flush_clwb)
			info->memmove_nodrain =
				memmove_nodrain_sse2_clwb_nobarrier;
		else
			ASSERT(0);

		info->memset_nodrain_eadr =
				memset_nodrain_sse2_eadr_nobarrier;
		if (info->flush == flush_clflush)
			info->memset_nodrain =
				memset_nodrain_sse2_clflush_nobarrier;
		else if (info->flush == flush_clflushopt)
			info->memset_nodrain =
				memset_nodrain_sse2_clflushopt_nobarrier;
		else if (info->flush == flush_clwb)
			info->memset_nodrain =
				memset_nodrain_sse2_clwb_nobarrier;
		else
			ASSERT(0);
	}

#else
	LOG(3, "sse2 disabled at build time");
#endif

}

/*
 * use_avx_memcpy_memset -- (internal) AVX detected, use it if possible
 */
static void
use_avx_memcpy_memset(struct pmem2_arch_info *info, enum memcpy_impl *impl,
		int wc_workaround)
{
#if AVX_AVAILABLE
	LOG(3, "avx supported");

	char *e = os_getenv("PMEM_AVX");
	if (e != NULL && strcmp(e, "0") == 0) {
		LOG(3, "PMEM_AVX set to 0");
		return;
	}

	LOG(3, "PMEM_AVX enabled");
	*impl = MEMCPY_AVX;

	if (wc_workaround) {
		info->memmove_nodrain_eadr =
				memmove_nodrain_avx_eadr_wcbarrier;
		if (info->flush == flush_clflush)
			info->memmove_nodrain =
				memmove_nodrain_avx_clflush_wcbarrier;
		else if (info->flush == flush_clflushopt)
			info->memmove_nodrain =
				memmove_nodrain_avx_clflushopt_wcbarrier;
		else if (info->flush == flush_clwb)
			info->memmove_nodrain =
				memmove_nodrain_avx_clwb_wcbarrier;
		else
			ASSERT(0);

		info->memset_nodrain_eadr =
				memset_nodrain_avx_eadr_wcbarrier;
		if (info->flush == flush_clflush)
			info->memset_nodrain =
				memset_nodrain_avx_clflush_wcbarrier;
		else if (info->flush == flush_clflushopt)
			info->memset_nodrain =
				memset_nodrain_avx_clflushopt_wcbarrier;
		else if (info->flush == flush_clwb)
			info->memset_nodrain =
				memset_nodrain_avx_clwb_wcbarrier;
		else
			ASSERT(0);
	} else {
		info->memmove_nodrain_eadr =
				memmove_nodrain_avx_eadr_nobarrier;
		if (info->flush == flush_clflush)
			info->memmove_nodrain =
				memmove_nodrain_avx_clflush_nobarrier;
		else if (info->flush == flush_clflushopt)
			info->memmove_nodrain =
				memmove_nodrain_avx_clflushopt_nobarrier;
		else if (info->flush == flush_clwb)
			info->memmove_nodrain =
				memmove_nodrain_avx_clwb_nobarrier;
		else
			ASSERT(0);

		info->memset_nodrain_eadr =
				memset_nodrain_avx_eadr_nobarrier;
		if (info->flush == flush_clflush)
			info->memset_nodrain =
				memset_nodrain_avx_clflush_nobarrier;
		else if (info->flush == flush_clflushopt)
			info->memset_nodrain =
				memset_nodrain_avx_clflushopt_nobarrier;
		else if (info->flush == flush_clwb)
			info->memset_nodrain =
				memset_nodrain_avx_clwb_nobarrier;
		else
			ASSERT(0);
	}
#else
	LOG(3, "avx supported, but disabled at build time");
#endif
}

/*
 * use_avx512f_memcpy_memset -- (internal) AVX512F detected, use it if possible
 */
static void
use_avx512f_memcpy_memset(struct pmem2_arch_info *info,
		enum memcpy_impl *impl)
{
#if AVX512F_AVAILABLE
	LOG(3, "avx512f supported");

	char *e = os_getenv("PMEM_AVX512F");
	if (e != NULL && strcmp(e, "0") == 0) {
		LOG(3, "PMEM_AVX512F set to 0");
		return;
	}

	LOG(3, "PMEM_AVX512F enabled");
	*impl = MEMCPY_AVX512F;

	info->memmove_nodrain_eadr = memmove_nodrain_avx512f_eadr;
	if (info->flush == flush_clflush)
		info->memmove_nodrain = memmove_nodrain_avx512f_clflush;
	else if (info->flush == flush_clflushopt)
		info->memmove_nodrain = memmove_nodrain_avx512f_clflushopt;
	else if (info->flush == flush_clwb)
		info->memmove_nodrain = memmove_nodrain_avx512f_clwb;
	else
		ASSERT(0);

	info->memset_nodrain_eadr = memset_nodrain_avx512f_eadr;
	if (info->flush == flush_clflush)
		info->memset_nodrain = memset_nodrain_avx512f_clflush;
	else if (info->flush == flush_clflushopt)
		info->memset_nodrain = memset_nodrain_avx512f_clflushopt;
	else if (info->flush == flush_clwb)
		info->memset_nodrain = memset_nodrain_avx512f_clwb;
	else
		ASSERT(0);
#else
	LOG(3, "avx512f supported, but disabled at build time");
#endif
}

/*
 * pmem_get_cpuinfo -- configure libpmem based on CPUID
 */
static void
pmem_cpuinfo_to_funcs(struct pmem2_arch_info *info, enum memcpy_impl *impl)
{
	LOG(3, NULL);

	if (is_cpu_clflush_present()) {
		LOG(3, "clflush supported");

		info->flush = flush_clflush;
		info->flush_has_builtin_fence = 1;
		info->fence = memory_barrier;
	}

	if (is_cpu_clflushopt_present()) {
		LOG(3, "clflushopt supported");

		char *e = os_getenv("PMEM_NO_CLFLUSHOPT");
		if (e && strcmp(e, "1") == 0) {
			LOG(3, "PMEM_NO_CLFLUSHOPT forced no clflushopt");
		} else {
			info->flush = flush_clflushopt;
			info->flush_has_builtin_fence = 0;
			info->fence = memory_barrier;
		}
	}

	if (is_cpu_clwb_present()) {
		LOG(3, "clwb supported");

		char *e = os_getenv("PMEM_NO_CLWB");
		if (e && strcmp(e, "1") == 0) {
			LOG(3, "PMEM_NO_CLWB forced no clwb");
		} else {
			info->flush = flush_clwb;
			info->flush_has_builtin_fence = 0;
			info->fence = memory_barrier;
		}
	}

	/*
	 * XXX Disable this work around for Intel CPUs with optimized
	 * WC eviction.
	 */
	int wc_workaround = is_cpu_genuine_intel();

	char *ptr = os_getenv("PMEM_WC_WORKAROUND");
	if (ptr) {
		if (strcmp(ptr, "1") == 0) {
			LOG(3, "WC workaround forced to 1");
			wc_workaround = 1;
		} else if (strcmp(ptr, "0") == 0) {
			LOG(3, "WC workaround forced to 0");
			wc_workaround = 0;
		} else {
			LOG(3, "incorrect value of PMEM_WC_WORKAROUND (%s)",
				ptr);
		}
	}
	LOG(3, "WC workaround = %d", wc_workaround);

	ptr = os_getenv("PMEM_NO_MOVNT");
	if (ptr && strcmp(ptr, "1") == 0) {
		LOG(3, "PMEM_NO_MOVNT forced no movnt");
	} else {
		use_sse2_memcpy_memset(info, impl, wc_workaround);

		if (is_cpu_avx_present())
			use_avx_memcpy_memset(info, impl, wc_workaround);

		if (is_cpu_avx512f_present())
			use_avx512f_memcpy_memset(info, impl);
	}
}

/*
 * pmem2_arch_init -- initialize architecture-specific list of pmem operations
 */
void
pmem2_arch_init(struct pmem2_arch_info *info)
{
	LOG(3, NULL);
	enum memcpy_impl impl = MEMCPY_INVALID;

	pmem_cpuinfo_to_funcs(info, &impl);

	/*
	 * For testing, allow overriding the default threshold
	 * for using non-temporal stores in pmem_memcpy_*(), pmem_memmove_*()
	 * and pmem_memset_*().
	 * It has no effect if movnt is not supported or disabled.
	 */
	const char *ptr = os_getenv("PMEM_MOVNT_THRESHOLD");
	if (ptr) {
		long long val = atoll(ptr);

		if (val < 0) {
			LOG(3, "Invalid PMEM_MOVNT_THRESHOLD");
		} else {
			LOG(3, "PMEM_MOVNT_THRESHOLD set to %zu", (size_t)val);
			Movnt_threshold = (size_t)val;
		}
	}

	if (info->flush == flush_clwb)
		LOG(3, "using clwb");
	else if (info->flush == flush_clflushopt)
		LOG(3, "using clflushopt");
	else if (info->flush == flush_clflush)
		LOG(3, "using clflush");
	else
		FATAL("invalid deep flush function address");

	if (impl == MEMCPY_AVX512F)
		LOG(3, "using movnt AVX512F");
	else if (impl == MEMCPY_AVX)
		LOG(3, "using movnt AVX");
	else if (impl == MEMCPY_SSE2)
		LOG(3, "using movnt SSE2");
}
