From 0eeba58a2a5273b5f9dd95d3e219a2d4ad003244 Mon Sep 17 00:00:00 2001
From: Honggang Li <honli@redhat.com>
Date: Mon, 11 Jan 2021 16:59:08 +0800
Subject: [PATCH] Check PCIe relaxed ordering compliant

Significantly performance degradation may be observed when PICe
relaxed ordering enabled over CPU which is not PCIe RO compliant.
Emit a warning message for such scenario.

https://github.com/linux-rdma/perftest/issues/116

Signed-off-by: Honggang Li <honli@redhat.com>
---
 README                    |  3 ++-
 configure.ac              |  3 +++
 perftest.spec             |  1 +
 src/perftest_parameters.c | 37 +++++++++++++++++++++++++++++++++++++
 4 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/README b/README
index 23f81cb..707d9df 100644
--- a/README
+++ b/README
@@ -108,7 +108,8 @@ Prerequisites:
 	(kernel module) matches libibverbs
 	(kernel module) matches librdmacm
 	(kernel module) matches libibumad
-	(kernel module) matches libmath (lm).
+	(kernel module) matches libmath (lm)
+	(linux kernel module) matches pciutils (lpci).
 	
 
 Server:		./<test name> <options>
diff --git a/configure.ac b/configure.ac
index 0a1cc41..66ec909 100755
--- a/configure.ac
+++ b/configure.ac
@@ -174,6 +174,8 @@ if [test $HAVE_SNIFFER = yes]; then
 fi
 
 if [test $IS_FREEBSD = no]; then
+	AC_CHECK_HEADERS([pci/pci.h],,[AC_MSG_ERROR([pciutils header files not found])])
+	AC_CHECK_LIB([pci], [pci_init], [LIBPCI=-lpci], AC_MSG_ERROR([libpci not found]))
 	CPU_IS_RO_COMPLIANT=yes
 	# Actually this is check for being affected by a known issue
 	# with Intel CPUs:
@@ -193,6 +195,7 @@ AC_TRY_LINK([
 AM_CONDITIONAL([HAVE_RO],[test "x$HAVE_RO" = "xyes"])
 if [test $HAVE_RO = yes] && [test "x$CPU_IS_RO_COMPLIANT" = "xyes"]; then
         AC_DEFINE([HAVE_RO], [1], [Enable Relaxed Ordering])
+	LIBS=$LIBS" -lpci"
 fi
 
 AC_TRY_LINK([#include <infiniband/verbs.h>],
diff --git a/perftest.spec b/perftest.spec
index 630b00b..b0b50da 100644
--- a/perftest.spec
+++ b/perftest.spec
@@ -8,6 +8,7 @@ Source:         http://www.openfabrics.org/downloads/%{name}-%{version}.tar.gz
 Url:            http://www.openfabrics.org
 BuildRoot:      %{_tmppath}/%{name}-%{version}-build
 BuildRequires:  libibverbs-devel librdmacm-devel libibumad-devel
+BuildRequires:  pciutils-devel
 
 %description
 gen3 uverbs microbenchmarks
diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c
index 185d7e3..77fe835 100755
--- a/src/perftest_parameters.c
+++ b/src/perftest_parameters.c
@@ -11,6 +11,10 @@
 #include "perftest_parameters.h"
 #include "raw_ethernet_resources.h"
 #include<math.h>
+#ifdef HAVE_RO
+#include <stdbool.h>
+#include <pci/pci.h>
+#endif
 #define MAC_LEN (17)
 #define ETHERTYPE_LEN (6)
 #define MAC_ARR_LEN (6)
@@ -163,6 +167,34 @@ static int get_cache_line_size()
 
 	return size;
 }
+#ifdef HAVE_RO
+/******************************************************************************
+  Check PCIe Relaxed Ordering
+
+  Stolen from https://github.com/pciutils/pciutils/blob/master/example.c
+ ******************************************************************************/
+static bool check_pcie_relaxed_ordering_compliant(void) {
+	struct pci_access *pacc;
+	struct pci_dev *dev;
+	bool cpu_is_RO_compliant = true;
+
+	pacc = pci_alloc();
+	pci_init(pacc);
+	pci_scan_bus(pacc);
+	for (dev = pacc->devices; dev && cpu_is_RO_compliant;
+	     dev = dev->next) {
+		pci_fill_info(dev,
+			      PCI_FILL_IDENT | PCI_FILL_BASES | PCI_FILL_CLASS);
+		/* https://lore.kernel.org/patchwork/patch/820922/ */
+		if ((dev->vendor_id == 0x8086) &&
+		    (((dev->device_id >= 0x6f01 && dev->device_id <= 0x6f0e) ||
+		      (dev->device_id >= 0x2f01 && dev->device_id <= 0x2f01))))
+			cpu_is_RO_compliant = false;
+	}
+	pci_cleanup(pacc);
+	return cpu_is_RO_compliant;
+}
+#endif
 /******************************************************************************
  *
  ******************************************************************************/
@@ -2873,6 +2905,11 @@ void ctx_print_test_info(struct perftest_parameters *user_param)
 	printf(" Connection type : %s\t\tUsing SRQ      : %s\n", connStr[user_param->connection_type], user_param->use_srq ? "ON"  : "OFF");
 	#ifdef HAVE_RO
 	printf(" PCIe relax order: %s\n", user_param->disable_pcir ? "OFF"  : "ON");
+	if ((check_pcie_relaxed_ordering_compliant() == false) &&
+	    (user_param->disable_pcir == 0)) {
+		printf(" WARNING: CPU is not PCIe relaxed ordering compliant.\n");
+		printf(" WARNING: You should disable PCIe RO with `--disable_pcie_relaxed` for both server and clinet.\n");
+	}
 	#else
 	printf(" PCIe relax order: %s\n", "Unsupported");
 	#endif
-- 
2.29.2