From 63c80c7692e55f634cbca6f67cc5c9cdef3a04d2 Mon Sep 17 00:00:00 2001

From: Honggang Li <honli@redhat.com>

Date: Mon, 28 Jun 2021 21:38:13 +0800

Subject: [PATCH] Revert "ucx: check supported transports and devices for

 setting priority"

This reverts commit c36d7459b6331c4da825cad5a64326e7c1a272aa.

---

 contrib/platform/mellanox/optimized.conf |   2 -

 ompi/mca/pml/ucx/pml_ucx_component.c     |  15 +-

 opal/mca/common/ucx/common_ucx.c         | 202 +----------------------

 opal/mca/common/ucx/common_ucx.h         |  15 --

 opal/mca/common/ucx/configure.m4         |   2 -

 5 files changed, 2 insertions(+), 234 deletions(-)

diff --git a/contrib/platform/mellanox/optimized.conf b/contrib/platform/mellanox/optimized.conf

index 543fd8d1e224..b86b37c9e2fa 100644

--- a/contrib/platform/mellanox/optimized.conf

+++ b/contrib/platform/mellanox/optimized.conf

@@ -61,8 +61,6 @@

 coll = ^ml

 hwloc_base_binding_policy = core

 btl = self

-pml_ucx_tls = any

-pml_ucx_devices = any

 # Basic behavior to smooth startup

 mca_base_component_show_load_errors = 0

 orte_abort_timeout = 10

diff --git a/ompi/mca/pml/ucx/pml_ucx_component.c b/ompi/mca/pml/ucx/pml_ucx_component.c

index 6aed6c41d11d..ed9cc6573e8e 100644

--- a/ompi/mca/pml/ucx/pml_ucx_component.c

+++ b/ompi/mca/pml/ucx/pml_ucx_component.c

@@ -107,26 +107,13 @@ static mca_pml_base_module_t*

 mca_pml_ucx_component_init(int* priority, bool enable_progress_threads,

                            bool enable_mpi_threads)

 {

-    opal_common_ucx_support_level_t support_level;

     int ret;

-    support_level = opal_common_ucx_support_level(ompi_pml_ucx.ucp_context);

-    if (support_level == OPAL_COMMON_UCX_SUPPORT_NONE) {

-        return NULL;

-    }

-

     if ( (ret = mca_pml_ucx_init(enable_mpi_threads)) != 0) {

         return NULL;

     }

-    /*

-     * If found supported devices - set to the configured (high) priority.

-     * Otherwise - Found only supported transports (which could be exposed by

-     *             unsupported devices), so set a priority lower than ob1.

-     */

-    *priority = (support_level == OPAL_COMMON_UCX_SUPPORT_DEVICE) ?

-                ompi_pml_ucx.priority : 19;

-    PML_UCX_VERBOSE(2, "returning priority %d", *priority);

+    *priority = ompi_pml_ucx.priority;

     return &ompi_pml_ucx.super;

 }

diff --git a/opal/mca/common/ucx/common_ucx.c b/opal/mca/common/ucx/common_ucx.c

index ac7a17d799a5..ae8e66877ab6 100644

--- a/opal/mca/common/ucx/common_ucx.c

+++ b/opal/mca/common/ucx/common_ucx.c

@@ -14,11 +14,8 @@

 #include "opal/mca/base/mca_base_framework.h"

 #include "opal/mca/pmix/pmix.h"

 #include "opal/memoryhooks/memory.h"

-#include "opal/util/argv.h"

 #include <ucm/api/ucm.h>

-#include <fnmatch.h>

-#include <stdio.h>

 /***********************************************************************/

@@ -28,8 +25,7 @@ opal_common_ucx_module_t opal_common_ucx = {

     .verbose             = 0,

     .progress_iterations = 100,

     .registered          = 0,

-    .opal_mem_hooks      = 0,

-    .tls                 = NULL

+    .opal_mem_hooks      = 0

 };

 static void opal_common_ucx_mem_release_cb(void *buf, size_t length,

@@ -40,15 +36,10 @@ static void opal_common_ucx_mem_release_cb(void *buf, size_t length,

 OPAL_DECLSPEC void opal_common_ucx_mca_var_register(const mca_base_component_t *component)

 {

-    static const char *default_tls     = "rc_verbs,ud_verbs,rc_mlx5,dc_mlx5,cuda_ipc,rocm_ipc";

-    static const char *default_devices = "mlx*";

     static int registered = 0;

     static int hook_index;

     static int verbose_index;

     static int progress_index;

-    static int tls_index;

-    static int devices_index;

-

     if (!registered) {

         verbose_index = mca_base_var_register("opal", "opal_common", "ucx", "verbose",

                                               "Verbose level of the UCX components",

@@ -69,29 +60,6 @@ OPAL_DECLSPEC void opal_common_ucx_mca_var_register(const mca_base_component_t *

                                            OPAL_INFO_LVL_3,

                                            MCA_BASE_VAR_SCOPE_LOCAL,

                                            &opal_common_ucx.opal_mem_hooks);

-

-        opal_common_ucx.tls  = malloc(sizeof(*opal_common_ucx.tls));

-        *opal_common_ucx.tls = strdup(default_tls);

-        tls_index = mca_base_var_register("opal", "opal_common", "ucx", "tls",

-                                          "List of UCX transports which should be supported on the system, to enable "

-                                          "selecting the UCX component. Special values: any (any available). "

-                                          "A '^' prefix negates the list. "

-                                          "For example, in order to exclude on shared memory and TCP transports, "

-                                          "please set to '^posix,sysv,self,tcp,cma,knem,xpmem'.",

-                                          MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,

-                                          OPAL_INFO_LVL_3,

-                                          MCA_BASE_VAR_SCOPE_LOCAL,

-                                          opal_common_ucx.tls);

-

-        opal_common_ucx.devices  = malloc(sizeof(*opal_common_ucx.devices));

-        *opal_common_ucx.devices = strdup(default_devices);

-        devices_index = mca_base_var_register("opal", "opal_common", "ucx", "devices",

-                                              "List of device driver pattern names, which, if supported by UCX, will "

-                                              "bump its priority above ob1. Special values: any (any available)",

-                                              MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,

-                                              OPAL_INFO_LVL_3,

-                                              MCA_BASE_VAR_SCOPE_LOCAL,

-                                              opal_common_ucx.devices);

         registered = 1;

     }

     if (component) {

@@ -107,14 +75,6 @@ OPAL_DECLSPEC void opal_common_ucx_mca_var_register(const mca_base_component_t *

                                       component->mca_type_name,

                                       component->mca_component_name,

                                       "opal_mem_hooks", 0);

-        mca_base_var_register_synonym(tls_index, component->mca_project_name,

-                                      component->mca_type_name,

-                                      component->mca_component_name,

-                                      "tls", 0);

-        mca_base_var_register_synonym(devices_index, component->mca_project_name,

-                                      component->mca_type_name,

-                                      component->mca_component_name,

-                                      "devices", 0);

     }

 }

@@ -163,166 +123,6 @@ OPAL_DECLSPEC void opal_common_ucx_mca_deregister(void)

     opal_output_close(opal_common_ucx.output);

 }

-#if HAVE_DECL_OPEN_MEMSTREAM

-static bool opal_common_ucx_check_device(const char *device_name, char **device_list)

-{

-    char sysfs_driver_link[PATH_MAX];

-    char driver_path[PATH_MAX];

-    char *ib_device_name;

-    char *driver_name;

-    char **list_item;

-    ssize_t ret;

-

-    /* mlx5_0:1 */

-    ret = sscanf(device_name, "%m[^:]%*d", &ib_device_name);

-    if (ret != 1) {

-        return false;

-    }

-

-    sysfs_driver_link[sizeof(sysfs_driver_link) - 1] = '\0';

-    snprintf(sysfs_driver_link, sizeof(sysfs_driver_link) - 1,

-             "/sys/class/infiniband/%s/device/driver", ib_device_name);

-    free(ib_device_name);

-

-    driver_path[sizeof(driver_path) - 1] = '\0';

-    ret = readlink(sysfs_driver_link, driver_path, sizeof(driver_path) - 1);

-    if (ret < 0) {

-        MCA_COMMON_UCX_VERBOSE(2, "readlink(%s) failed: %s", sysfs_driver_link,

-                               strerror(errno));

-        return false;

-    }

-

-    driver_name = basename(driver_path);

-    for (list_item = device_list; *list_item != NULL; ++list_item) {

-        if (!fnmatch(*list_item, driver_name, 0)) {

-            MCA_COMMON_UCX_VERBOSE(2, "driver '%s' matched by '%s'",

-                                   driver_path, *list_item);

-            return true;

-        }

-    }

-

-    return false;

-}

-#endif

-

-OPAL_DECLSPEC opal_common_ucx_support_level_t

-opal_common_ucx_support_level(ucp_context_h context)

-{

-    opal_common_ucx_support_level_t support_level = OPAL_COMMON_UCX_SUPPORT_NONE;

-    static const char *support_level_names[] = {

-        [OPAL_COMMON_UCX_SUPPORT_NONE]      = "none",

-        [OPAL_COMMON_UCX_SUPPORT_TRANSPORT] = "transports only",

-        [OPAL_COMMON_UCX_SUPPORT_DEVICE]    = "transports and devices"

-    };

-#if HAVE_DECL_OPEN_MEMSTREAM

-    char *rsc_tl_name, *rsc_device_name;

-    char **tl_list, **device_list, **list_item;

-    bool is_any_tl, is_any_device;

-    bool found_tl, negate;

-    char line[128];

-    FILE *stream;

-    char *buffer;

-    size_t size;

-    int ret;

-#endif

-

-    is_any_tl     = !strcmp(*opal_common_ucx.tls, "any");

-    is_any_device = !strcmp(*opal_common_ucx.devices, "any");

-

-    /* Check for special value "any" */

-    if (is_any_tl && is_any_device) {

-        MCA_COMMON_UCX_VERBOSE(1, "ucx is enabled on any transport or device",

-                               *opal_common_ucx.tls);

-        support_level = OPAL_COMMON_UCX_SUPPORT_DEVICE;

-        goto out;

-    }

-

-#if HAVE_DECL_OPEN_MEMSTREAM

-    /* Split transports list */

-    negate  = ('^' == (*opal_common_ucx.tls)[0]);

-    tl_list = opal_argv_split(*opal_common_ucx.tls + (negate ? 1 : 0), ',');

-    if (tl_list == NULL) {

-        MCA_COMMON_UCX_VERBOSE(1, "failed to split tl list '%s', ucx is disabled",

-                               *opal_common_ucx.tls);

-        goto out;

-    }

-

-    /* Split devices list */

-    device_list = opal_argv_split(*opal_common_ucx.devices, ',');

-    if (device_list == NULL) {

-        MCA_COMMON_UCX_VERBOSE(1, "failed to split devices list '%s', ucx is disabled",

-                               *opal_common_ucx.devices);

-        goto out_free_tl_list;

-    }

-

-    /* Open memory stream to dump UCX information to */

-    stream = open_memstream(&buffer, &size);

-    if (stream == NULL) {

-        MCA_COMMON_UCX_VERBOSE(1, "failed to open memory stream for ucx info (%s), "

-                               "ucx is disabled", strerror(errno));

-        goto out_free_device_list;

-    }

-

-    /* Print ucx transports information to the memory stream */

-    ucp_context_print_info(context, stream);

-

-    /* Rewind and read transports/devices list from the stream */

-    fseek(stream, 0, SEEK_SET);

-    while ((support_level != OPAL_COMMON_UCX_SUPPORT_DEVICE) &&

-           (fgets(line, sizeof(line), stream) != NULL)) {

-        rsc_tl_name = NULL;

-        ret = sscanf(line,

-                     /* "# resource 6  :  md 5  dev 4  flags -- rc_verbs/mlx5_0:1" */

-                     "# resource %*d : md %*d dev %*d flags -- %m[^/ \n\r]/%m[^/ \n\r]",

-                     &rsc_tl_name, &rsc_device_name);

-        if (ret != 2) {

-            free(rsc_tl_name);

-            continue;

-        }

-

-        /* Check if 'rsc_tl_name' is found  provided list */

-        found_tl = is_any_tl;

-        for (list_item = tl_list; !found_tl && (*list_item != NULL); ++list_item) {

-            found_tl = !strcmp(*list_item, rsc_tl_name);

-        }

-

-        /* Check if the transport has a match (either positive or negative) */

-        assert(!(is_any_tl && negate));

-        if (found_tl != negate) {

-            if (is_any_device ||

-                opal_common_ucx_check_device(rsc_device_name, device_list)) {

-                MCA_COMMON_UCX_VERBOSE(2, "%s/%s: matched both transport and device list",

-                                    rsc_tl_name, rsc_device_name);

-                support_level = OPAL_COMMON_UCX_SUPPORT_DEVICE;

-            } else {

-                MCA_COMMON_UCX_VERBOSE(2, "%s/%s: matched transport list but not device list",

-                                    rsc_tl_name, rsc_device_name);

-                support_level = OPAL_COMMON_UCX_SUPPORT_TRANSPORT;

-            }

-        } else {

-            MCA_COMMON_UCX_VERBOSE(2, "%s/%s: did not match transport list",

-                                   rsc_tl_name, rsc_device_name);

-        }

-

-        free(rsc_device_name);

-        free(rsc_tl_name);

-    }

-

-    MCA_COMMON_UCX_VERBOSE(2, "support level is %s", support_level_names[support_level]);

-    fclose(stream);

-    free(buffer);

-

-out_free_device_list:

-    opal_argv_free(device_list);

-out_free_tl_list:

-    opal_argv_free(tl_list);

-out:

-#else

-    MCA_COMMON_UCX_VERBOSE(2, "open_memstream() was not found, ucx is disabled");

-#endif

-    return support_level;

-}

-

 void opal_common_ucx_empty_complete_cb(void *request, ucs_status_t status)

 {

 }

diff --git a/opal/mca/common/ucx/common_ucx.h b/opal/mca/common/ucx/common_ucx.h

index 92cdd738ef98..202131ac8907 100644

--- a/opal/mca/common/ucx/common_ucx.h

+++ b/opal/mca/common/ucx/common_ucx.h

@@ -88,8 +88,6 @@ typedef struct opal_common_ucx_module {

     int  progress_iterations;

     int  registered;

     bool opal_mem_hooks;

-    char **tls;

-    char **devices;

 } opal_common_ucx_module_t;

 typedef struct opal_common_ucx_del_proc {

@@ -97,23 +95,10 @@ typedef struct opal_common_ucx_del_proc {

     size_t   vpid;

 } opal_common_ucx_del_proc_t;

-typedef enum {

-    /* No supported transports found (according to configured list of supported

-       transports) */

-    OPAL_COMMON_UCX_SUPPORT_NONE,

-

-    /* Have supported transports but not supported devices */

-    OPAL_COMMON_UCX_SUPPORT_TRANSPORT,

-

-    /* Have both supported transports and supported devices */

-    OPAL_COMMON_UCX_SUPPORT_DEVICE,

-} opal_common_ucx_support_level_t;

-

 extern opal_common_ucx_module_t opal_common_ucx;

 OPAL_DECLSPEC void opal_common_ucx_mca_register(void);

 OPAL_DECLSPEC void opal_common_ucx_mca_deregister(void);

-OPAL_DECLSPEC opal_common_ucx_support_level_t opal_common_ucx_support_level(ucp_context_h context);

 OPAL_DECLSPEC void opal_common_ucx_mca_proc_added(void);

 OPAL_DECLSPEC void opal_common_ucx_empty_complete_cb(void *request, ucs_status_t status);

 OPAL_DECLSPEC int opal_common_ucx_mca_pmix_fence(ucp_worker_h worker);

diff --git a/opal/mca/common/ucx/configure.m4 b/opal/mca/common/ucx/configure.m4

index af8628a889c6..27e07c2005b2 100644

--- a/opal/mca/common/ucx/configure.m4

+++ b/opal/mca/common/ucx/configure.m4

@@ -18,8 +18,6 @@ AC_DEFUN([MCA_opal_common_ucx_CONFIG],[

                [common_ucx_happy="yes"],

                [common_ucx_happy="no"])

-    AC_CHECK_DECLS([open_memstream], [], [], [[#include <stdio.h>]])

-

     AS_IF([test "$common_ucx_happy" = "yes"],

           [$1],

           [$2])

-- 

2.31.1