From eef9685245d172a80e9a5dfd830942824e7d40b4 Mon Sep 17 00:00:00 2001
From: Dan Williams <dan.j.williams@intel.com>
Date: Thu, 14 Jul 2022 10:02:55 -0700
Subject: [PATCH 186/217] cxl/test: Checkout region setup/teardown

Exercise the fundamental region provisioning sysfs mechanisms of discovering
available DPA capacity, allocating DPA to a region, and programming HDM
decoders.

Link: https://lore.kernel.org/r/165781817516.1555691.3557156570639615515.stgit@dwillia2-xfh.jf.intel.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Vishal Verma <vishal.l.verma@intel.com>
---
 test/cxl-region-sysfs.sh | 122 +++++++++++++++++++++++++++++++++++++++
 test/meson.build         |   2 +
 2 files changed, 124 insertions(+)
 create mode 100644 test/cxl-region-sysfs.sh

diff --git a/test/cxl-region-sysfs.sh b/test/cxl-region-sysfs.sh
new file mode 100644
index 0000000..2582edb
--- /dev/null
+++ b/test/cxl-region-sysfs.sh
@@ -0,0 +1,122 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (C) 2022 Intel Corporation. All rights reserved.
+
+. $(dirname $0)/common
+
+rc=1
+
+set -ex
+
+trap 'err $LINENO' ERR
+
+check_prereq "jq"
+
+modprobe -r cxl_test
+modprobe cxl_test
+udevadm settle
+
+# THEORY OF OPERATION: Create a x8 interleave across the pmem capacity
+# of the 8 endpoints defined by cxl_test, commit the decoders (which
+# just stubs out the actual hardware programming aspect, but updates the
+# driver state), and then tear it all down again. As with other cxl_test
+# tests if the CXL topology in tools/testing/cxl/test/cxl.c ever changes
+# then the paired update must be made to this test.
+
+# find the root decoder that spans both test host-bridges and support pmem
+decoder=$($CXL list -b cxl_test -D -d root | jq -r ".[] |
+	  select(.pmem_capable == true) |
+	  select(.nr_targets == 2) |
+	  .decoder")
+
+# find the memdevs mapped by that decoder
+readarray -t mem < <($CXL list -M -d $decoder | jq -r ".[].memdev")
+
+# ask cxl reserve-dpa to allocate pmem capacity from each of those memdevs
+readarray -t endpoint < <($CXL reserve-dpa -t pmem ${mem[*]} -s $((256<<20)) |
+			  jq -r ".[] | .decoder.decoder")
+
+# instantiate an empty region
+region=$(cat /sys/bus/cxl/devices/$decoder/create_pmem_region)
+echo $region > /sys/bus/cxl/devices/$decoder/create_pmem_region
+uuidgen > /sys/bus/cxl/devices/$region/uuid
+
+# setup interleave geometry
+nr_targets=${#endpoint[@]}
+echo $nr_targets > /sys/bus/cxl/devices/$region/interleave_ways
+g=$(cat /sys/bus/cxl/devices/$decoder/interleave_granularity)
+echo $g > /sys/bus/cxl/devices/$region/interleave_granularity
+echo $((nr_targets * (256<<20))) > /sys/bus/cxl/devices/$region/size
+
+# grab the list of memdevs grouped by host-bridge interleave position
+port_dev0=$($CXL list -T -d $decoder | jq -r ".[] |
+	    .targets | .[] | select(.position == 0) | .target")
+port_dev1=$($CXL list -T -d $decoder | jq -r ".[] |
+	    .targets | .[] | select(.position == 1) | .target")
+readarray -t mem_sort0 < <($CXL list -M -p $port_dev0 | jq -r ".[] | .memdev")
+readarray -t mem_sort1 < <($CXL list -M -p $port_dev1 | jq -r ".[] | .memdev")
+
+# TODO: add a cxl list option to list memdevs in valid region provisioning
+# order, hardcode for now.
+mem_sort=()
+mem_sort[0]=${mem_sort0[0]}
+mem_sort[1]=${mem_sort1[0]}
+mem_sort[2]=${mem_sort0[2]}
+mem_sort[3]=${mem_sort1[2]}
+mem_sort[4]=${mem_sort0[1]}
+mem_sort[5]=${mem_sort1[1]}
+mem_sort[6]=${mem_sort0[3]}
+mem_sort[7]=${mem_sort1[3]}
+
+# TODO: use this alternative memdev ordering to validate a negative test for
+# specifying invalid positions of memdevs
+#mem_sort[2]=${mem_sort0[0]}
+#mem_sort[1]=${mem_sort1[0]}
+#mem_sort[0]=${mem_sort0[2]}
+#mem_sort[3]=${mem_sort1[2]}
+#mem_sort[4]=${mem_sort0[1]}
+#mem_sort[5]=${mem_sort1[1]}
+#mem_sort[6]=${mem_sort0[3]}
+#mem_sort[7]=${mem_sort1[3]}
+
+# re-generate the list of endpoint decoders in region position programming order
+endpoint=()
+for i in ${mem_sort[@]}
+do
+	readarray -O ${#endpoint[@]} -t endpoint < <($CXL list -Di -d endpoint -m $i | jq -r ".[] |
+						     select(.mode == \"pmem\") | .decoder")
+done
+
+# attach all endpoint decoders to the region
+pos=0
+for i in ${endpoint[@]}
+do
+	echo $i > /sys/bus/cxl/devices/$region/target$pos
+	pos=$((pos+1))
+done
+echo "$region added ${#endpoint[@]} targets: ${endpoint[@]}"
+
+# walk up the topology and commit all decoders
+echo 1 > /sys/bus/cxl/devices/$region/commit
+
+# walk down the topology and de-commit all decoders
+echo 0 > /sys/bus/cxl/devices/$region/commit
+
+# remove endpoints from the region
+pos=0
+for i in ${endpoint[@]}
+do
+	echo "" > /sys/bus/cxl/devices/$region/target$pos
+	pos=$((pos+1))
+done
+
+# release DPA capacity
+readarray -t endpoint < <($CXL free-dpa -t pmem ${mem[*]} |
+			  jq -r ".[] | .decoder.decoder")
+echo "$region released ${#endpoint[@]} targets: ${endpoint[@]}"
+
+# validate no WARN or lockdep report during the run
+log=$(journalctl -r -k --since "-$((SECONDS+1))s")
+grep -q "Call Trace" <<< $log && err "$LINENO"
+
+modprobe -r cxl_test
diff --git a/test/meson.build b/test/meson.build
index 210dcb0..3203d9c 100644
--- a/test/meson.build
+++ b/test/meson.build
@@ -151,6 +151,7 @@ max_extent = find_program('max_available_extent_ns.sh')
 pfn_meta_errors = find_program('pfn-meta-errors.sh')
 track_uuid = find_program('track-uuid.sh')
 cxl_topo = find_program('cxl-topology.sh')
+cxl_sysfs = find_program('cxl-region-sysfs.sh')
 
 tests = [
   [ 'libndctl',               libndctl,		  'ndctl' ],
@@ -176,6 +177,7 @@ tests = [
   [ 'pfn-meta-errors.sh',     pfn_meta_errors,	  'ndctl' ],
   [ 'track-uuid.sh',          track_uuid,	  'ndctl' ],
   [ 'cxl-topology.sh',	      cxl_topo,		  'cxl'   ],
+  [ 'cxl-region-sysfs.sh',    cxl_sysfs,	  'cxl'   ],
 ]
 
 if get_option('destructive').enabled()
-- 
2.27.0