From 62b970d5e9edbdd68dc006193b0e606fb7ae7cdd Mon Sep 17 00:00:00 2001

From: Tomas Jelinek <tojeline@redhat.com>

Date: Thu, 2 Jul 2020 15:18:29 +0200

Subject: [PATCH 1/3] upgrade CIB schema for on-fail=demote

---

 pcs/lib/cib/resource/operations.py            |   9 +-

 pcs/lib/commands/remote_node.py               |  73 +++--

 pcs/lib/commands/resource.py                  | 269 ++++++++++--------

 pcs/resource.py                               |  51 +++-

 pcs_test/resources/cib-empty-3.3.xml          |   2 +-

 pcs_test/resources/cib-empty-3.4.xml          |   2 +-

 .../tier0/lib/cib/test_resource_operations.py |   9 +-

 .../remote_node/test_node_add_remote.py       |  54 +++-

 .../commands/resource/test_resource_create.py | 237 ++++++++++++++-

 pcs_test/tier1/cib_resource/test_create.py    |   2 +-

 pcs_test/tier1/legacy/test_resource.py        |  23 ++

 pcs_test/tools/misc.py                        |   6 +

 pcsd/capabilities.xml                         |  42 +++

 13 files changed, 594 insertions(+), 185 deletions(-)

diff --git a/pcs/lib/cib/resource/operations.py b/pcs/lib/cib/resource/operations.py

index 131e0a49..79d00685 100644

--- a/pcs/lib/cib/resource/operations.py

+++ b/pcs/lib/cib/resource/operations.py

@@ -39,13 +39,14 @@ ATTRIBUTES = [

 ]

 ON_FAIL_VALUES = [

-    "ignore",

     "block",

-    "stop",

-    "restart",

-    "standby",

+    "demote",

     "fence",

+    "ignore",

+    "restart",

     "restart-container",

+    "standby",

+    "stop",

 ]

 BOOLEAN_VALUES = [

diff --git a/pcs/lib/commands/remote_node.py b/pcs/lib/commands/remote_node.py

index 6a2656a5..575e8044 100644

--- a/pcs/lib/commands/remote_node.py

+++ b/pcs/lib/commands/remote_node.py

@@ -1,3 +1,10 @@

+from typing import (

+    Iterable,

+    Mapping,

+    Optional,

+    Union,

+)

+

 from pcs import settings

 from pcs.common import reports

 from pcs.common.file import RawFileError

@@ -13,6 +20,10 @@ from pcs.lib.cib.tools import (

     ElementSearcher,

     get_resources,

 )

+

+# TODO lib.commands should never import each other. This is to be removed when

+# the 'resource create' commands are overhauled.

+from pcs.lib.commands.resource import get_required_cib_version_for_primitive

 from pcs.lib.communication.nodes import (

     DistributeFiles,

     GetHostInfo,

@@ -24,6 +35,7 @@ from pcs.lib.communication.tools import (

     run as run_com,

     run_and_raise,

 )

+from pcs.lib.corosync.config_facade import ConfigFacade as CorosyncConfigFacade

 from pcs.lib.env import LibraryEnvironment

 from pcs.lib.errors import LibraryError

 from pcs.lib.file.instance import FileInstance

@@ -33,6 +45,9 @@ from pcs.lib.pacemaker import state

 from pcs.lib.pacemaker.live import remove_node

+WaitType = Union[None, bool, int]

+

+

 def _reports_skip_new_node(new_node_name, reason_type):

     assert reason_type in {"unreachable", "not_live_cib"}

     return [

@@ -220,19 +235,19 @@ def _ensure_resource_running(env: LibraryEnvironment, resource_id):

 def node_add_remote(

-    env,

-    node_name,

-    node_addr,

-    operations,

-    meta_attributes,

-    instance_attributes,

-    skip_offline_nodes=False,

-    allow_incomplete_distribution=False,

-    allow_pacemaker_remote_service_fail=False,

-    allow_invalid_operation=False,

-    allow_invalid_instance_attributes=False,

-    use_default_operations=True,

-    wait=False,

+    env: LibraryEnvironment,

+    node_name: str,

+    node_addr: Optional[str],

+    operations: Iterable[Mapping[str, str]],

+    meta_attributes: Mapping[str, str],

+    instance_attributes: Mapping[str, str],

+    skip_offline_nodes: bool = False,

+    allow_incomplete_distribution: bool = False,

+    allow_pacemaker_remote_service_fail: bool = False,

+    allow_invalid_operation: bool = False,

+    allow_invalid_instance_attributes: bool = False,

+    use_default_operations: bool = True,

+    wait: WaitType = False,

 ):

     # pylint: disable=too-many-arguments

     # pylint: disable=too-many-branches

@@ -241,34 +256,36 @@ def node_add_remote(

     """

     create an ocf:pacemaker:remote resource and use it as a remote node

-    LibraryEnvironment env -- provides all for communication with externals

-    string node_name -- the name of the new node

-    mixed node_addr -- the address of the new node or None for default

-    list of dict operations -- attributes for each entered operation

-    dict meta_attributes -- attributes for primitive/meta_attributes

-    dict instance_attributes -- attributes for primitive/instance_attributes

-    bool skip_offline_nodes -- if True, ignore when some nodes are offline

-    bool allow_incomplete_distribution -- if True, allow this command to

+    env -- provides all for communication with externals

+    node_name -- the name of the new node

+    node_addr -- the address of the new node or None for default

+    operations -- attributes for each entered operation

+    meta_attributes -- attributes for primitive/meta_attributes

+    instance_attributes -- attributes for primitive/instance_attributes

+    skip_offline_nodes -- if True, ignore when some nodes are offline

+    allow_incomplete_distribution -- if True, allow this command to

         finish successfully even if file distribution did not succeed

-    bool allow_pacemaker_remote_service_fail -- if True, allow this command to

+    allow_pacemaker_remote_service_fail -- if True, allow this command to

         finish successfully even if starting/enabling pacemaker_remote did not

         succeed

-    bool allow_invalid_operation -- if True, allow to use operations that

+    allow_invalid_operation -- if True, allow to use operations that

         are not listed in a resource agent metadata

-    bool allow_invalid_instance_attributes -- if True, allow to use instance

+    allow_invalid_instance_attributes -- if True, allow to use instance

         attributes that are not listed in a resource agent metadata and allow to

         omit required instance_attributes

-    bool use_default_operations -- if True, add operations specified in

+    use_default_operations -- if True, add operations specified in

         a resource agent metadata to the resource

-    mixed wait -- a flag for controlling waiting for pacemaker idle mechanism

+    wait -- a flag for controlling waiting for pacemaker idle mechanism

     """

     env.ensure_wait_satisfiable(wait)

     report_processor = env.report_processor

-    cib = env.get_cib()

+    cib = env.get_cib(

+        minimal_version=get_required_cib_version_for_primitive(operations)

+    )

     id_provider = IdProvider(cib)

     if env.is_cib_live:

-        corosync_conf = env.get_corosync_conf()

+        corosync_conf: Optional[CorosyncConfigFacade] = env.get_corosync_conf()

     else:

         corosync_conf = None

         report_processor.report(

diff --git a/pcs/lib/commands/resource.py b/pcs/lib/commands/resource.py

index 75826c9d..db4b7bb3 100644

--- a/pcs/lib/commands/resource.py

+++ b/pcs/lib/commands/resource.py

@@ -61,6 +61,9 @@ from pcs.lib.resource_agent import (

 from pcs.lib.validate import ValueTimeInterval

+WaitType = Union[None, bool, int]

+

+

 @contextmanager

 def resource_environment(

     env,

@@ -262,44 +265,43 @@ def _get_required_cib_version_for_container(

 def create(

-    env,

-    resource_id,

-    resource_agent_name,

-    operation_list,

-    meta_attributes,

-    instance_attributes,

-    allow_absent_agent=False,

-    allow_invalid_operation=False,

-    allow_invalid_instance_attributes=False,

-    use_default_operations=True,

-    ensure_disabled=False,

-    wait=False,

-    allow_not_suitable_command=False,

+    env: LibraryEnvironment,

+    resource_id: str,

+    resource_agent_name: str,

+    operation_list: Iterable[Mapping[str, str]],

+    meta_attributes: Mapping[str, str],

+    instance_attributes: Mapping[str, str],

+    allow_absent_agent: bool = False,

+    allow_invalid_operation: bool = False,

+    allow_invalid_instance_attributes: bool = False,

+    use_default_operations: bool = True,

+    ensure_disabled: bool = False,

+    wait: WaitType = False,

+    allow_not_suitable_command: bool = False,

 ):

     # pylint: disable=too-many-arguments, too-many-locals

     """

-    Create resource in a cib.

+    Create a primitive resource in a cib.

-    LibraryEnvironment env provides all for communication with externals

-    string resource_id is identifier of resource

-    string resource_agent_name contains name for the identification of agent

-    list of dict operation_list contains attributes for each entered operation

-    dict meta_attributes contains attributes for primitive/meta_attributes

-    dict instance_attributes contains attributes for

-        primitive/instance_attributes

-    bool allow_absent_agent is a flag for allowing agent that is not installed

+    env -- provides all for communication with externals

+    resource_id -- is identifier of resource

+    resource_agent_name -- contains name for the identification of agent

+    operation_list -- contains attributes for each entered operation

+    meta_attributes -- contains attributes for primitive/meta_attributes

+    instance_attributes -- contains attributes for primitive/instance_attributes

+    allow_absent_agent -- is a flag for allowing agent that is not installed

         in a system

-    bool allow_invalid_operation is a flag for allowing to use operations that

+    allow_invalid_operation -- is a flag for allowing to use operations that

         are not listed in a resource agent metadata

-    bool allow_invalid_instance_attributes is a flag for allowing to use

+    allow_invalid_instance_attributes -- is a flag for allowing to use

         instance attributes that are not listed in a resource agent metadata

         or for allowing to not use the instance_attributes that are required in

         resource agent metadata

-    bool use_default_operations is a flag for stopping stopping of adding

+    use_default_operations -- is a flag for stopping stopping of adding

         default cib operations (specified in a resource agent)

-    bool ensure_disabled is flag that keeps resource in target-role "Stopped"

-    mixed wait is flag for controlling waiting for pacemaker idle mechanism

-    bool allow_not_suitable_command -- flag for FORCE_NOT_SUITABLE_COMMAND

+    ensure_disabled -- is flag that keeps resource in target-role "Stopped"

+    wait -- is flag for controlling waiting for pacemaker idle mechanism

+    allow_not_suitable_command -- flag for FORCE_NOT_SUITABLE_COMMAND

     """

     resource_agent = get_agent(

         env.report_processor,

@@ -315,6 +317,9 @@ def create(

             ensure_disabled

             or resource.common.are_meta_disabled(meta_attributes)

         ),

+        required_cib_version=get_required_cib_version_for_primitive(

+            operation_list

+        ),

     ) as resources_section:

         id_provider = IdProvider(resources_section)

         _check_special_cases(

@@ -345,46 +350,45 @@ def create(

 def create_as_clone(

-    env,

-    resource_id,

-    resource_agent_name,

-    operation_list,

-    meta_attributes,

-    instance_attributes,

-    clone_meta_options,

-    allow_absent_agent=False,

-    allow_invalid_operation=False,

-    allow_invalid_instance_attributes=False,

-    use_default_operations=True,

-    ensure_disabled=False,

-    wait=False,

-    allow_not_suitable_command=False,

+    env: LibraryEnvironment,

+    resource_id: str,

+    resource_agent_name: str,

+    operation_list: Iterable[Mapping[str, str]],

+    meta_attributes: Mapping[str, str],

+    instance_attributes: Mapping[str, str],

+    clone_meta_options: Mapping[str, str],

+    allow_absent_agent: bool = False,

+    allow_invalid_operation: bool = False,

+    allow_invalid_instance_attributes: bool = False,

+    use_default_operations: bool = True,

+    ensure_disabled: bool = False,

+    wait: WaitType = False,

+    allow_not_suitable_command: bool = False,

 ):

     # pylint: disable=too-many-arguments, too-many-locals

     """

-    Create resource in a clone

+    Create a primitive resource in a clone

-    LibraryEnvironment env provides all for communication with externals

-    string resource_id is identifier of resource

-    string resource_agent_name contains name for the identification of agent

-    list of dict operation_list contains attributes for each entered operation

-    dict meta_attributes contains attributes for primitive/meta_attributes

-    dict instance_attributes contains attributes for

-        primitive/instance_attributes

-    dict clone_meta_options contains attributes for clone/meta_attributes

-    bool allow_absent_agent is a flag for allowing agent that is not installed

+    env -- provides all for communication with externals

+    resource_id -- is identifier of resource

+    resource_agent_name -- contains name for the identification of agent

+    operation_list -- contains attributes for each entered operation

+    meta_attributes -- contains attributes for primitive/meta_attributes

+    instance_attributes -- contains attributes for primitive/instance_attributes

+    clone_meta_options -- contains attributes for clone/meta_attributes

+    allow_absent_agent -- is a flag for allowing agent that is not installed

         in a system

-    bool allow_invalid_operation is a flag for allowing to use operations that

+    allow_invalid_operation -- is a flag for allowing to use operations that

         are not listed in a resource agent metadata

-    bool allow_invalid_instance_attributes is a flag for allowing to use

+    allow_invalid_instance_attributes -- is a flag for allowing to use

         instance attributes that are not listed in a resource agent metadata

         or for allowing to not use the instance_attributes that are required in

         resource agent metadata

-    bool use_default_operations is a flag for stopping stopping of adding

+    use_default_operations -- is a flag for stopping stopping of adding

         default cib operations (specified in a resource agent)

-    bool ensure_disabled is flag that keeps resource in target-role "Stopped"

-    mixed wait is flag for controlling waiting for pacemaker idle mechanism

-    bool allow_not_suitable_command -- flag for FORCE_NOT_SUITABLE_COMMAND

+    ensure_disabled -- is flag that keeps resource in target-role "Stopped"

+    wait -- is flag for controlling waiting for pacemaker idle mechanism

+    allow_not_suitable_command -- flag for FORCE_NOT_SUITABLE_COMMAND

     """

     resource_agent = get_agent(

         env.report_processor,

@@ -401,6 +405,9 @@ def create_as_clone(

             or resource.common.are_meta_disabled(meta_attributes)

             or resource.common.is_clone_deactivated_by_meta(clone_meta_options)

         ),

+        required_cib_version=get_required_cib_version_for_primitive(

+            operation_list

+        ),

     ) as resources_section:

         id_provider = IdProvider(resources_section)

         _check_special_cases(

@@ -437,49 +444,50 @@ def create_as_clone(

 def create_in_group(

-    env,

-    resource_id,

-    resource_agent_name,

-    group_id,

-    operation_list,

-    meta_attributes,

-    instance_attributes,

-    allow_absent_agent=False,

-    allow_invalid_operation=False,

-    allow_invalid_instance_attributes=False,

-    use_default_operations=True,

-    ensure_disabled=False,

-    adjacent_resource_id=None,

-    put_after_adjacent=False,

-    wait=False,

-    allow_not_suitable_command=False,

+    env: LibraryEnvironment,

+    resource_id: str,

+    resource_agent_name: str,

+    group_id: str,

+    operation_list: Iterable[Mapping[str, str]],

+    meta_attributes: Mapping[str, str],

+    instance_attributes: Mapping[str, str],

+    allow_absent_agent: bool = False,

+    allow_invalid_operation: bool = False,

+    allow_invalid_instance_attributes: bool = False,

+    use_default_operations: bool = True,

+    ensure_disabled: bool = False,

+    adjacent_resource_id: Optional[str] = None,

+    put_after_adjacent: bool = False,

+    wait: WaitType = False,

+    allow_not_suitable_command: bool = False,

 ):

     # pylint: disable=too-many-arguments, too-many-locals

     """

     Create resource in a cib and put it into defined group

-    LibraryEnvironment env provides all for communication with externals

-    string resource_id is identifier of resource

-    string resource_agent_name contains name for the identification of agent

-    string group_id is identificator for group to put primitive resource inside

-    list of dict operation_list contains attributes for each entered operation

-    dict meta_attributes contains attributes for primitive/meta_attributes

-    bool allow_absent_agent is a flag for allowing agent that is not installed

+    env -- provides all for communication with externals

+    resource_id -- is identifier of resource

+    resource_agent_name -- contains name for the identification of agent

+    group_id -- is identificator for group to put primitive resource inside

+    operation_list -- contains attributes for each entered operation

+    meta_attributes -- contains attributes for primitive/meta_attributes

+    instance_attributes -- contains attributes for primitive/instance_attributes

+    allow_absent_agent -- is a flag for allowing agent that is not installed

         in a system

-    bool allow_invalid_operation is a flag for allowing to use operations that

+    allow_invalid_operation -- is a flag for allowing to use operations that

         are not listed in a resource agent metadata

-    bool allow_invalid_instance_attributes is a flag for allowing to use

+    allow_invalid_instance_attributes -- is a flag for allowing to use

         instance attributes that are not listed in a resource agent metadata

         or for allowing to not use the instance_attributes that are required in

         resource agent metadata

-    bool use_default_operations is a flag for stopping stopping of adding

+    use_default_operations -- is a flag for stopping stopping of adding

         default cib operations (specified in a resource agent)

-    bool ensure_disabled is flag that keeps resource in target-role "Stopped"

-    string adjacent_resource_id identify neighbor of a newly created resource

-    bool put_after_adjacent is flag to put a newly create resource befor/after

+    ensure_disabled -- is flag that keeps resource in target-role "Stopped"

+    adjacent_resource_id -- identify neighbor of a newly created resource

+    put_after_adjacent -- is flag to put a newly create resource befor/after

         adjacent resource

-    mixed wait is flag for controlling waiting for pacemaker idle mechanism

-    bool allow_not_suitable_command -- flag for FORCE_NOT_SUITABLE_COMMAND

+    wait -- is flag for controlling waiting for pacemaker idle mechanism

+    allow_not_suitable_command -- flag for FORCE_NOT_SUITABLE_COMMAND

     """

     resource_agent = get_agent(

         env.report_processor,

@@ -495,6 +503,9 @@ def create_in_group(

             ensure_disabled

             or resource.common.are_meta_disabled(meta_attributes)

         ),

+        required_cib_version=get_required_cib_version_for_primitive(

+            operation_list

+        ),

     ) as resources_section:

         id_provider = IdProvider(resources_section)

         _check_special_cases(

@@ -532,48 +543,48 @@ def create_in_group(

 def create_into_bundle(

-    env,

-    resource_id,

-    resource_agent_name,

-    operation_list,

-    meta_attributes,

-    instance_attributes,

-    bundle_id,

-    allow_absent_agent=False,

-    allow_invalid_operation=False,

-    allow_invalid_instance_attributes=False,

-    use_default_operations=True,

-    ensure_disabled=False,

-    wait=False,

-    allow_not_suitable_command=False,

-    allow_not_accessible_resource=False,

+    env: LibraryEnvironment,

+    resource_id: str,

+    resource_agent_name: str,

+    operation_list: Iterable[Mapping[str, str]],

+    meta_attributes: Mapping[str, str],

+    instance_attributes: Mapping[str, str],

+    bundle_id: str,

+    allow_absent_agent: bool = False,

+    allow_invalid_operation: bool = False,

+    allow_invalid_instance_attributes: bool = False,

+    use_default_operations: bool = True,

+    ensure_disabled: bool = False,

+    wait: WaitType = False,

+    allow_not_suitable_command: bool = False,

+    allow_not_accessible_resource: bool = False,

 ):

     # pylint: disable=too-many-arguments, too-many-locals

     """

     Create a new resource in a cib and put it into an existing bundle

-    LibraryEnvironment env provides all for communication with externals

-    string resource_id is identifier of resource

-    string resource_agent_name contains name for the identification of agent

-    list of dict operation_list contains attributes for each entered operation

-    dict meta_attributes contains attributes for primitive/meta_attributes

-    dict instance_attributes contains attributes for

+    env -- provides all for communication with externals

+    resource_id -- is identifier of resource

+    resource_agent_name -- contains name for the identification of agent

+    operation_list -- contains attributes for each entered operation

+    meta_attributes -- contains attributes for primitive/meta_attributes

+    instance_attributes -- contains attributes for

         primitive/instance_attributes

-    string bundle_id is id of an existing bundle to put the created resource in

-    bool allow_absent_agent is a flag for allowing agent that is not installed

+    bundle_id -- is id of an existing bundle to put the created resource in

+    allow_absent_agent -- is a flag for allowing agent that is not installed

         in a system

-    bool allow_invalid_operation is a flag for allowing to use operations that

+    allow_invalid_operation -- is a flag for allowing to use operations that

         are not listed in a resource agent metadata

-    bool allow_invalid_instance_attributes is a flag for allowing to use

+    allow_invalid_instance_attributes -- is a flag for allowing to use

         instance attributes that are not listed in a resource agent metadata

         or for allowing to not use the instance_attributes that are required in

         resource agent metadata

-    bool use_default_operations is a flag for stopping stopping of adding

+    use_default_operations -- is a flag for stopping stopping of adding

         default cib operations (specified in a resource agent)

-    bool ensure_disabled is flag that keeps resource in target-role "Stopped"

-    mixed wait is flag for controlling waiting for pacemaker idle mechanism

-    bool allow_not_suitable_command -- flag for FORCE_NOT_SUITABLE_COMMAND

-    bool allow_not_accessible_resource -- flag for

+    ensure_disabled -- is flag that keeps resource in target-role "Stopped"

+    wait -- is flag for controlling waiting for pacemaker idle mechanism

+    allow_not_suitable_command -- flag for FORCE_NOT_SUITABLE_COMMAND

+    allow_not_accessible_resource -- flag for

         FORCE_RESOURCE_IN_BUNDLE_NOT_ACCESSIBLE

     """

     resource_agent = get_agent(

@@ -582,6 +593,11 @@ def create_into_bundle(

         resource_agent_name,

         allow_absent_agent,

     )

+    required_cib_version = get_required_cib_version_for_primitive(

+        operation_list

+    )

+    if not required_cib_version:

+        required_cib_version = Version(2, 8, 0)

     with resource_environment(

         env,

         wait,

@@ -590,7 +606,7 @@ def create_into_bundle(

             ensure_disabled

             or resource.common.are_meta_disabled(meta_attributes)

         ),

-        required_cib_version=Version(2, 8, 0),

+        required_cib_version=required_cib_version,

     ) as resources_section:

         id_provider = IdProvider(resources_section)

         _check_special_cases(

@@ -1070,9 +1086,7 @@ def disable_simulate(

 def enable(

-    env: LibraryEnvironment,

-    resource_or_tag_ids: Iterable[str],

-    wait: Optional[Union[bool, int]],

+    env: LibraryEnvironment, resource_or_tag_ids: Iterable[str], wait: WaitType,

 ):

     """

     Allow specified resources to be started by the cluster

@@ -1689,3 +1703,12 @@ def _find_resources_expand_tags_or_raise(

     return resource.common.expand_tags_to_resources(

         resources_section, resource_or_tag_el_list,

     )

+

+

+def get_required_cib_version_for_primitive(

+    op_list: Iterable[Mapping[str, str]]

+) -> Optional[Version]:

+    for op in op_list:

+        if op.get("on-fail", "") == "demote":

+            return Version(3, 4, 0)

+    return None

diff --git a/pcs/resource.py b/pcs/resource.py

index e835fc99..9a3bd0ee 100644

--- a/pcs/resource.py

+++ b/pcs/resource.py

@@ -355,6 +355,21 @@ def resource_op_add_cmd(lib, argv, modifiers):

     if not argv:

         raise CmdLineInputError()

     res_id = argv.pop(0)

+

+    # Check if we need to upgrade cib schema.

+    # To do that, argv must be parsed, which is duplication of parsing in

+    # resource_operation_add. But we need to upgrade the cib first before

+    # calling that function. Hopefully, this will be fixed in the new pcs

+    # architecture.

+

+    # argv[0] is an operation name

+    op_properties = utils.convert_args_to_tuples(argv[1:])

+    for key, value in op_properties:

+        if key == "on-fail" and value == "demote":

+            utils.checkAndUpgradeCIB(3, 4, 0)

+            break

+

+    # add the requested operation

     utils.replace_cib_configuration(

         resource_operation_add(utils.get_cib_dom(), res_id, argv)

     )

@@ -895,8 +910,6 @@ def resource_update(lib, args, modifiers, deal_with_guest_change=True):

     if len(args) < 2:

         raise CmdLineInputError()

     res_id = args.pop(0)

-    cib_xml = utils.get_cib()

-    dom = utils.get_cib_dom(cib_xml=cib_xml)

     # Extract operation arguments

     ra_values, op_values, meta_values = parse_resource_options(args)

@@ -907,6 +920,28 @@ def resource_update(lib, args, modifiers, deal_with_guest_change=True):

         wait_timeout = utils.validate_wait_get_timeout()

         wait = True

+    # Check if we need to upgrade cib schema.

+    # To do that, argv must be parsed, which is duplication of parsing below.

+    # But we need to upgrade the cib first before calling that function.

+    # Hopefully, this will be fixed in the new pcs architecture.

+

+    cib_upgraded = False

+    for op_argv in op_values:

+        if cib_upgraded:

+            break

+        if len(op_argv) < 2:

+            continue

+        # argv[0] is an operation name

+        op_vars = utils.convert_args_to_tuples(op_argv[1:])

+        for k, v in op_vars:

+            if k == "on-fail" and v == "demote":

+                utils.checkAndUpgradeCIB(3, 4, 0)

+                cib_upgraded = True

+                break

+

+    cib_xml = utils.get_cib()

+    dom = utils.get_cib_dom(cib_xml=cib_xml)

+

     resource = utils.dom_get_resource(dom, res_id)

     if not resource:

         clone = utils.dom_get_clone(dom, res_id)

@@ -994,21 +1029,21 @@ def resource_update(lib, args, modifiers, deal_with_guest_change=True):

     else:

         operations = operations[0]

-    for element in op_values:

-        if not element:

+    for op_argv in op_values:

+        if not op_argv:

             continue

-        op_name = element[0]

+        op_name = op_argv[0]

         if op_name.find("=") != -1:

             utils.err(

                 "%s does not appear to be a valid operation action" % op_name

             )

-        if len(element) < 2:

+        if len(op_argv) < 2:

             continue

         op_role = ""

-        op_vars = utils.convert_args_to_tuples(element[1:])

+        op_vars = utils.convert_args_to_tuples(op_argv[1:])

         for k, v in op_vars:

             if k == "role":

@@ -1032,7 +1067,7 @@ def resource_update(lib, args, modifiers, deal_with_guest_change=True):

         dom = resource_operation_add(

             dom,

             res_id,

-            element,

+            op_argv,

             validate_strict=False,

             before_op=updating_op_before,

         )

diff --git a/pcs_test/resources/cib-empty-3.3.xml b/pcs_test/resources/cib-empty-3.3.xml

index 3a44fe08..4de94b6e 100644

--- a/pcs_test/resources/cib-empty-3.3.xml

+++ b/pcs_test/resources/cib-empty-3.3.xml

@@ -1,4 +1,4 @@

-<cib epoch="557" num_updates="122" admin_epoch="0" validate-with="pacemaker-3.3" crm_feature_set="3.1.0" update-origin="rh7-3" update-client="crmd" cib-last-written="Thu Aug 23 16:49:17 2012" have-quorum="0" dc-uuid="2">

+<cib epoch="557" num_updates="122" admin_epoch="0" validate-with="pacemaker-3.3" crm_feature_set="3.4.0" update-origin="rh7-3" update-client="crmd" cib-last-written="Thu Aug 23 16:49:17 2012" have-quorum="0" dc-uuid="2">

   <configuration>

     <crm_config/>

     <nodes>

diff --git a/pcs_test/resources/cib-empty-3.4.xml b/pcs_test/resources/cib-empty-3.4.xml

index dcd4ff44..e677462d 100644

--- a/pcs_test/resources/cib-empty-3.4.xml

+++ b/pcs_test/resources/cib-empty-3.4.xml

@@ -1,4 +1,4 @@

-<cib epoch="557" num_updates="122" admin_epoch="0" validate-with="pacemaker-3.4" crm_feature_set="3.1.0" update-origin="rh7-3" update-client="crmd" cib-last-written="Thu Aug 23 16:49:17 2012" have-quorum="0" dc-uuid="2">

+<cib epoch="557" num_updates="122" admin_epoch="0" validate-with="pacemaker-3.4" crm_feature_set="3.4.0" update-origin="rh7-3" update-client="crmd" cib-last-written="Thu Aug 23 16:49:17 2012" have-quorum="0" dc-uuid="2">

   <configuration>

     <crm_config/>

     <nodes>

diff --git a/pcs_test/tier0/lib/cib/test_resource_operations.py b/pcs_test/tier0/lib/cib/test_resource_operations.py

index 5e556cf4..e5be7a54 100644

--- a/pcs_test/tier0/lib/cib/test_resource_operations.py

+++ b/pcs_test/tier0/lib/cib/test_resource_operations.py

@@ -316,13 +316,14 @@ class ValidateOperation(TestCase):

                     option_value="b",

                     option_name="on-fail",

                     allowed_values=[

-                        "ignore",

                         "block",

-                        "stop",

-                        "restart",

-                        "standby",

+                        "demote",

                         "fence",

+                        "ignore",

+                        "restart",

                         "restart-container",

+                        "standby",

+                        "stop",

                     ],

                     cannot_be_empty=False,

                     forbidden_characters=None,

diff --git a/pcs_test/tier0/lib/commands/remote_node/test_node_add_remote.py b/pcs_test/tier0/lib/commands/remote_node/test_node_add_remote.py

index 725ec68a..4da3fa0a 100644

--- a/pcs_test/tier0/lib/commands/remote_node/test_node_add_remote.py

+++ b/pcs_test/tier0/lib/commands/remote_node/test_node_add_remote.py

@@ -118,7 +118,7 @@ FIXTURE_RESOURCES_TEMPLATE = """