From ec14b8ed9cb4264333b80b4361171b1b529c58f3 Mon Sep 17 00:00:00 2001
From: Eduardo Otubo <otubo@redhat.com>
Date: Tue, 3 Nov 2020 12:11:45 +0100
Subject: [PATCH 3/5] Add config modules for controlling IBM PowerVM RMC.
(#584)
RH-Author: Eduardo Terrell Ferrari Otubo (eterrell)
RH-MergeRequest: 16: Add config modules for controlling IBM PowerVM RMC. (#584)
RH-Commit: [1/1] 734e2c48d323af31aa36abefae346ef62ba3ef5d (eterrell/cloud-init)
RH-Bugzilla: 1894014
commit f99d4f96b00a9cfec1c721d364cbfd728674e5dc
Author: Aman306 <45781773+Aman306@users.noreply.github.com>
Date: Wed Oct 28 23:36:09 2020 +0530
Add config modules for controlling IBM PowerVM RMC. (#584)
Reliable Scalable Cluster Technology (RSCT) is a set of software
components that together provide a comprehensive clustering
environment(RAS features) for IBM PowerVM based virtual machines. RSCT
includes the Resource Monitoring and Control (RMC) subsystem. RMC is a
generalized framework used for managing, monitoring, and manipulating
resources. RMC runs as a daemon process on individual machines and needs
creation of unique node id and restarts during VM boot.
LP: #1895979
Co-authored-by: Scott Moser <smoser@brickies.net>
Conflicts:
* Calls to module subp.* are replaced by old calls to util.* since the
patch that groups subp.* calls into its own module are introduced after
19.4 release - and it's a huge reafctoring not worth the cherry-pick.
Signed-off-by: Eduardo Otubo <otubo@redhat.com>
---
cloudinit/config/cc_refresh_rmc_and_interface.py | 158 +++++++++++++++++++++
cloudinit/config/cc_reset_rmc.py | 142 ++++++++++++++++++
config/cloud.cfg.tmpl | 2 +
.../test_handler_refresh_rmc_and_interface.py | 109 ++++++++++++++
4 files changed, 411 insertions(+)
create mode 100644 cloudinit/config/cc_refresh_rmc_and_interface.py
create mode 100644 cloudinit/config/cc_reset_rmc.py
create mode 100644 tests/unittests/test_handler/test_handler_refresh_rmc_and_interface.py
diff --git a/cloudinit/config/cc_refresh_rmc_and_interface.py b/cloudinit/config/cc_refresh_rmc_and_interface.py
new file mode 100644
index 0000000..07050c4
--- /dev/null
+++ b/cloudinit/config/cc_refresh_rmc_and_interface.py
@@ -0,0 +1,158 @@
+# (c) Copyright IBM Corp. 2020 All Rights Reserved
+#
+# Author: Aman Kumar Sinha <amansi26@in.ibm.com>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+"""
+Refresh IPv6 interface and RMC
+------------------------------
+**Summary:** Ensure Network Manager is not managing IPv6 interface
+
+This module is IBM PowerVM Hypervisor specific
+
+Reliable Scalable Cluster Technology (RSCT) is a set of software components
+that together provide a comprehensive clustering environment(RAS features)
+for IBM PowerVM based virtual machines. RSCT includes the Resource
+Monitoring and Control (RMC) subsystem. RMC is a generalized framework used
+for managing, monitoring, and manipulating resources. RMC runs as a daemon
+process on individual machines and needs creation of unique node id and
+restarts during VM boot.
+More details refer
+https://www.ibm.com/support/knowledgecenter/en/SGVKBA_3.2/admin/bl503_ovrv.htm
+
+This module handles
+- Refreshing RMC
+- Disabling NetworkManager from handling IPv6 interface, as IPv6 interface
+ is used for communication between RMC daemon and PowerVM hypervisor.
+
+**Internal name:** ``cc_refresh_rmc_and_interface``
+
+**Module frequency:** per always
+
+**Supported distros:** RHEL
+
+"""
+
+from cloudinit import log as logging
+from cloudinit.settings import PER_ALWAYS
+from cloudinit import util
+from cloudinit import netinfo
+
+import errno
+
+frequency = PER_ALWAYS
+
+LOG = logging.getLogger(__name__)
+# Ensure that /opt/rsct/bin has been added to standard PATH of the
+# distro. The symlink to rmcctrl is /usr/sbin/rsct/bin/rmcctrl .
+RMCCTRL = 'rmcctrl'
+
+
+def handle(name, _cfg, _cloud, _log, _args):
+ if not util.which(RMCCTRL):
+ LOG.debug("No '%s' in path, disabled", RMCCTRL)
+ return
+
+ LOG.debug(
+ 'Making the IPv6 up explicitly. '
+ 'Ensuring IPv6 interface is not being handled by NetworkManager '
+ 'and it is restarted to re-establish the communication with '
+ 'the hypervisor')
+
+ ifaces = find_ipv6_ifaces()
+
+ # Setting NM_CONTROLLED=no for IPv6 interface
+ # making it down and up
+
+ if len(ifaces) == 0:
+ LOG.debug("Did not find any interfaces with ipv6 addresses.")
+ else:
+ for iface in ifaces:
+ refresh_ipv6(iface)
+ disable_ipv6(sysconfig_path(iface))
+ restart_network_manager()
+
+
+def find_ipv6_ifaces():
+ info = netinfo.netdev_info()
+ ifaces = []
+ for iface, data in info.items():
+ if iface == "lo":
+ LOG.debug('Skipping localhost interface')
+ if len(data.get("ipv4", [])) != 0:
+ # skip this interface, as it has ipv4 addrs
+ continue
+ ifaces.append(iface)
+ return ifaces
+
+
+def refresh_ipv6(interface):
+ # IPv6 interface is explicitly brought up, subsequent to which the
+ # RMC services are restarted to re-establish the communication with
+ # the hypervisor.
+ util.subp(['ip', 'link', 'set', interface, 'down'])
+ util.subp(['ip', 'link', 'set', interface, 'up'])
+
+
+def sysconfig_path(iface):
+ return '/etc/sysconfig/network-scripts/ifcfg-' + iface
+
+
+def restart_network_manager():
+ util.subp(['systemctl', 'restart', 'NetworkManager'])
+
+
+def disable_ipv6(iface_file):
+ # Ensuring that the communication b/w the hypervisor and VM is not
+ # interrupted due to NetworkManager. For this purpose, as part of
+ # this function, the NM_CONTROLLED is explicitly set to No for IPV6
+ # interface and NetworkManager is restarted.
+ try:
+ contents = util.load_file(iface_file)
+ except IOError as e:
+ if e.errno == errno.ENOENT:
+ LOG.debug("IPv6 interface file %s does not exist\n",
+ iface_file)
+ else:
+ raise e
+
+ if 'IPV6INIT' not in contents:
+ LOG.debug("Interface file %s did not have IPV6INIT", iface_file)
+ return
+
+ LOG.debug("Editing interface file %s ", iface_file)
+
+ # Dropping any NM_CONTROLLED or IPV6 lines from IPv6 interface file.
+ lines = contents.splitlines()
+ lines = [line for line in lines if not search(line)]
+ lines.append("NM_CONTROLLED=no")
+
+ with open(iface_file, "w") as fp:
+ fp.write("\n".join(lines) + "\n")
+
+
+def search(contents):
+ # Search for any NM_CONTROLLED or IPV6 lines in IPv6 interface file.
+ return(
+ contents.startswith("IPV6ADDR") or
+ contents.startswith("IPADDR6") or
+ contents.startswith("IPV6INIT") or
+ contents.startswith("NM_CONTROLLED"))
+
+
+def refresh_rmc():
+ # To make a healthy connection between RMC daemon and hypervisor we
+ # refresh RMC. With refreshing RMC we are ensuring that making IPv6
+ # down and up shouldn't impact communication between RMC daemon and
+ # hypervisor.
+ # -z : stop Resource Monitoring & Control subsystem and all resource
+ # managers, but the command does not return control to the user
+ # until the subsystem and all resource managers are stopped.
+ # -s : start Resource Monitoring & Control subsystem.
+ try:
+ util.subp([RMCCTRL, '-z'])
+ util.subp([RMCCTRL, '-s'])
+ except Exception:
+ util.logexc(LOG, 'Failed to refresh the RMC subsystem.')
+ raise
diff --git a/cloudinit/config/cc_reset_rmc.py b/cloudinit/config/cc_reset_rmc.py
new file mode 100644
index 0000000..68373ad
--- /dev/null
+++ b/cloudinit/config/cc_reset_rmc.py
@@ -0,0 +1,142 @@
+# (c) Copyright IBM Corp. 2020 All Rights Reserved
+#
+# Author: Aman Kumar Sinha <amansi26@in.ibm.com>
+#
+# This file is part of cloud-init. See LICENSE file for license information.
+
+
+"""
+Reset RMC
+------------
+**Summary:** reset rsct node id
+
+Reset RMC module is IBM PowerVM Hypervisor specific
+
+Reliable Scalable Cluster Technology (RSCT) is a set of software components,
+that together provide a comprehensive clustering environment (RAS features)
+for IBM PowerVM based virtual machines. RSCT includes the Resource monitoring
+and control (RMC) subsystem. RMC is a generalized framework used for managing,
+monitoring, and manipulating resources. RMC runs as a daemon process on
+individual machines and needs creation of unique node id and restarts
+during VM boot.
+More details refer
+https://www.ibm.com/support/knowledgecenter/en/SGVKBA_3.2/admin/bl503_ovrv.htm
+
+This module handles
+- creation of the unique RSCT node id to every instance/virtual machine
+ and ensure once set, it isn't changed subsequently by cloud-init.
+ In order to do so, it restarts RSCT service.
+
+Prerequisite of using this module is to install RSCT packages.
+
+**Internal name:** ``cc_reset_rmc``
+
+**Module frequency:** per instance
+
+**Supported distros:** rhel, sles and ubuntu
+
+"""
+import os
+
+from cloudinit import log as logging
+from cloudinit.settings import PER_INSTANCE
+from cloudinit import util
+
+frequency = PER_INSTANCE
+
+# RMCCTRL is expected to be in system PATH (/opt/rsct/bin)
+# The symlink for RMCCTRL and RECFGCT are
+# /usr/sbin/rsct/bin/rmcctrl and
+# /usr/sbin/rsct/install/bin/recfgct respectively.
+RSCT_PATH = '/opt/rsct/install/bin'
+RMCCTRL = 'rmcctrl'
+RECFGCT = 'recfgct'
+
+LOG = logging.getLogger(__name__)
+
+NODE_ID_FILE = '/etc/ct_node_id'
+
+
+def handle(name, _cfg, cloud, _log, _args):
+ # Ensuring node id has to be generated only once during first boot
+ if cloud.datasource.platform_type == 'none':
+ LOG.debug('Skipping creation of new ct_node_id node')
+ return
+
+ if not os.path.isdir(RSCT_PATH):
+ LOG.debug("module disabled, RSCT_PATH not present")
+ return
+
+ orig_path = os.environ.get('PATH')
+ try:
+ add_path(orig_path)
+ reset_rmc()
+ finally:
+ if orig_path:
+ os.environ['PATH'] = orig_path
+ else:
+ del os.environ['PATH']
+
+
+def reconfigure_rsct_subsystems():
+ # Reconfigure the RSCT subsystems, which includes removing all RSCT data
+ # under the /var/ct directory, generating a new node ID, and making it
+ # appear as if the RSCT components were just installed
+ try:
+ out = util.subp([RECFGCT])[0]
+ LOG.debug(out.strip())
+ return out
+ except util.ProcessExecutionError:
+ util.logexc(LOG, 'Failed to reconfigure the RSCT subsystems.')
+ raise
+
+
+def get_node_id():
+ try:
+ fp = util.load_file(NODE_ID_FILE)
+ node_id = fp.split('\n')[0]
+ return node_id
+ except Exception:
+ util.logexc(LOG, 'Failed to get node ID from file %s.' % NODE_ID_FILE)
+ raise
+
+
+def add_path(orig_path):
+ # Adding the RSCT_PATH to env standard path
+ # So thet cloud init automatically find and
+ # run RECFGCT to create new node_id.
+ suff = ":" + orig_path if orig_path else ""
+ os.environ['PATH'] = RSCT_PATH + suff
+ return os.environ['PATH']
+
+
+def rmcctrl():
+ # Stop the RMC subsystem and all resource managers so that we can make
+ # some changes to it
+ try:
+ return util.subp([RMCCTRL, '-z'])
+ except Exception:
+ util.logexc(LOG, 'Failed to stop the RMC subsystem.')
+ raise
+
+
+def reset_rmc():
+ LOG.debug('Attempting to reset RMC.')
+
+ node_id_before = get_node_id()
+ LOG.debug('Node ID at beginning of module: %s', node_id_before)
+
+ # Stop the RMC subsystem and all resource managers so that we can make
+ # some changes to it
+ rmcctrl()
+ reconfigure_rsct_subsystems()
+
+ node_id_after = get_node_id()
+ LOG.debug('Node ID at end of module: %s', node_id_after)
+
+ # Check if new node ID is generated or not
+ # by comparing old and new node ID
+ if node_id_after == node_id_before:
+ msg = 'New node ID did not get generated.'
+ LOG.error(msg)
+ raise Exception(msg)
diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl
index 87c37ba..52a259c 100644
--- a/config/cloud.cfg.tmpl
+++ b/config/cloud.cfg.tmpl
@@ -121,6 +121,8 @@ cloud_final_modules:
- mcollective
{% endif %}
- salt-minion
+ - reset_rmc
+ - refresh_rmc_and_interface
- rightscale_userdata
- scripts-vendor
- scripts-per-once
diff --git a/tests/unittests/test_handler/test_handler_refresh_rmc_and_interface.py b/tests/unittests/test_handler/test_handler_refresh_rmc_and_interface.py
new file mode 100644
index 0000000..0c35710
--- /dev/null
+++ b/tests/unittests/test_handler/test_handler_refresh_rmc_and_interface.py
@@ -0,0 +1,109 @@
+from cloudinit.config import cc_refresh_rmc_and_interface as ccrmci
+
+from cloudinit import util
+
+from cloudinit.tests import helpers as t_help
+from cloudinit.tests.helpers import mock
+
+from textwrap import dedent
+import logging
+
+LOG = logging.getLogger(__name__)
+MPATH = "cloudinit.config.cc_refresh_rmc_and_interface"
+NET_INFO = {
+ 'lo': {'ipv4': [{'ip': '127.0.0.1',
+ 'bcast': '', 'mask': '255.0.0.0',
+ 'scope': 'host'}],
+ 'ipv6': [{'ip': '::1/128',
+ 'scope6': 'host'}], 'hwaddr': '',
+ 'up': 'True'},
+ 'env2': {'ipv4': [{'ip': '8.0.0.19',
+ 'bcast': '8.0.0.255', 'mask': '255.255.255.0',
+ 'scope': 'global'}],
+ 'ipv6': [{'ip': 'fe80::f896:c2ff:fe81:8220/64',
+ 'scope6': 'link'}], 'hwaddr': 'fa:96:c2:81:82:20',
+ 'up': 'True'},
+ 'env3': {'ipv4': [{'ip': '90.0.0.14',
+ 'bcast': '90.0.0.255', 'mask': '255.255.255.0',
+ 'scope': 'global'}],
+ 'ipv6': [{'ip': 'fe80::f896:c2ff:fe81:8221/64',
+ 'scope6': 'link'}], 'hwaddr': 'fa:96:c2:81:82:21',
+ 'up': 'True'},
+ 'env4': {'ipv4': [{'ip': '9.114.23.7',
+ 'bcast': '9.114.23.255', 'mask': '255.255.255.0',
+ 'scope': 'global'}],
+ 'ipv6': [{'ip': 'fe80::f896:c2ff:fe81:8222/64',
+ 'scope6': 'link'}], 'hwaddr': 'fa:96:c2:81:82:22',
+ 'up': 'True'},
+ 'env5': {'ipv4': [],
+ 'ipv6': [{'ip': 'fe80::9c26:c3ff:fea4:62c8/64',
+ 'scope6': 'link'}], 'hwaddr': '42:20:86:df:fa:4c',
+ 'up': 'True'}}
+
+
+class TestRsctNodeFile(t_help.CiTestCase):
+ def test_disable_ipv6_interface(self):
+ """test parsing of iface files."""
+ fname = self.tmp_path("iface-eth5")
+ util.write_file(fname, dedent("""\
+ BOOTPROTO=static
+ DEVICE=eth5
+ HWADDR=42:20:86:df:fa:4c
+ IPV6INIT=yes
+ IPADDR6=fe80::9c26:c3ff:fea4:62c8/64
+ IPV6ADDR=fe80::9c26:c3ff:fea4:62c8/64
+ NM_CONTROLLED=yes
+ ONBOOT=yes
+ STARTMODE=auto
+ TYPE=Ethernet
+ USERCTL=no
+ """))
+
+ ccrmci.disable_ipv6(fname)
+ self.assertEqual(dedent("""\
+ BOOTPROTO=static
+ DEVICE=eth5
+ HWADDR=42:20:86:df:fa:4c
+ ONBOOT=yes
+ STARTMODE=auto
+ TYPE=Ethernet
+ USERCTL=no
+ NM_CONTROLLED=no
+ """), util.load_file(fname))
+
+ @mock.patch(MPATH + '.refresh_rmc')
+ @mock.patch(MPATH + '.restart_network_manager')
+ @mock.patch(MPATH + '.disable_ipv6')
+ @mock.patch(MPATH + '.refresh_ipv6')
+ @mock.patch(MPATH + '.netinfo.netdev_info')
+ @mock.patch(MPATH + '.util.which')
+ def test_handle(self, m_refresh_rmc,
+ m_netdev_info, m_refresh_ipv6, m_disable_ipv6,
+ m_restart_nm, m_which):
+ """Basic test of handle."""
+ m_netdev_info.return_value = NET_INFO
+ m_which.return_value = '/opt/rsct/bin/rmcctrl'
+ ccrmci.handle(
+ "refresh_rmc_and_interface", None, None, None, None)
+ self.assertEqual(1, m_netdev_info.call_count)
+ m_refresh_ipv6.assert_called_with('env5')
+ m_disable_ipv6.assert_called_with(
+ '/etc/sysconfig/network-scripts/ifcfg-env5')
+ self.assertEqual(1, m_restart_nm.call_count)
+ self.assertEqual(1, m_refresh_rmc.call_count)
+
+ @mock.patch(MPATH + '.netinfo.netdev_info')
+ def test_find_ipv6(self, m_netdev_info):
+ """find_ipv6_ifaces parses netdev_info returning those with ipv6"""
+ m_netdev_info.return_value = NET_INFO
+ found = ccrmci.find_ipv6_ifaces()
+ self.assertEqual(['env5'], found)
+
+ @mock.patch(MPATH + '.util.subp')
+ def test_refresh_ipv6(self, m_subp):
+ """refresh_ipv6 should ip down and up the interface."""
+ iface = "myeth0"
+ ccrmci.refresh_ipv6(iface)
+ m_subp.assert_has_calls([
+ mock.call(['ip', 'link', 'set', iface, 'down']),
+ mock.call(['ip', 'link', 'set', iface, 'up'])])
--
1.8.3.1