diff -uNr a/heartbeat/SAPHana b/heartbeat/SAPHana

--- a/heartbeat/SAPHana	2016-10-14 10:09:56.479051279 +0200

+++ b/heartbeat/SAPHana	2016-10-14 10:29:23.990066292 +0200

@@ -2,8 +2,8 @@

 #

 # SAPHana

 #

-# Description:  Manages two single SAP HANA Instance in System Replication

-#               Planned: do also manage scale-up scenarios

+# Description:  Manages two SAP HANA Databases in System Replication

+#               Planned: do also manage scale-out scenarios

 #               currently the SAPHana is dependent of the analysis of

 #               SAPHanaTopology

 #               For supported scenarios please read the README file provided

@@ -16,7 +16,7 @@

 # Support:      linux@sap.com

 # License:      GNU General Public License (GPL)

 # Copyright:    (c) 2013,2014 SUSE Linux Products GmbH

-# Copyright:    (c) 2015      SUSE Linux GmbH

+#               (c) 2015-2016 SUSE Linux GmbH

 #

 # An example usage:

 #      See usage() function below for more details...

@@ -29,12 +29,13 @@

 #   OCF_RESKEY_INSTANCE_PROFILE (optional, well known directories will be searched by default)

 #   OCF_RESKEY_PREFER_SITE_TAKEOVER (optional, default is no)

 #   OCF_RESKEY_DUPLICATE_PRIMARY_TIMEOUT (optional, time difference needed between two last-primary-tiemstampe (lpt))

-#   OCF_RESKEY_SAPHanaFilter    (optional, should only be set if been told by support or for debugging purposes)

+#   OCF_RESKEY_SAPHanaFilter    (outdated, replaced by cluster property hana_${sid}_glob_filter)

 #

 #

 #######################################################################

 #

 # Initialization:

+SAPHanaVersion="0.152.17"

 timeB=$(date '+%s')

 : ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}

@@ -43,6 +44,12 @@

 #

 #######################################################################

 #

+log_attributes=false

+if ocf_is_true "$log_attributes"; then

+    log_attr_file="/var/log/fhATTRIBUTES"

+else

+    log_attr_file="/dev/null"

+fi

 HANA_STATE_PRIMARY=0

 HANA_STATE_SECONDARY=1

@@ -107,7 +114,7 @@

     cat <<-EOF

     usage: $0 ($methods)

-    $0 manages a SAP HANA Instance as an HA resource.

+    $0 manages two SAP HANA databases (scale-up) in system replication.

     The 'start'        operation starts the HANA instance or bring the "clone instance" to a WAITING status

     The 'stop'         operation stops the HANA instance

@@ -145,15 +152,14 @@

 <resource-agent name="SAPHana">

-<version>0.151.1</version>

+<version>$SAPHanaVersion</version>

-<shortdesc lang="en">Manages two SAP HANA instances in system replication (SR).</shortdesc>

+<shortdesc lang="en">Manages two SAP HANA database systems in system replication (SR).</shortdesc>

 <longdesc lang="en">

-The SAPHanaSR resource agent manages two SAP Hana instances (databases) which are configured

-in system replication. This first version is limited to the scale-up scenario. Scale-Out is

-not supported in this version.

+The SAPHanaSR resource agent manages two SAP HANA database systems which are configured

+in system replication. SAPHana supports Scale-Up scenarios.

-Managing the two SAP HANA instances means that the resource agent controls the start/stop of the

+Managing the two SAP HANA database systems means that the resource agent controls the start/stop of the

 instances. In addition the resource agent is able to monitor the SAP HANA databases to check their

 availability on landscape host configuration level. For this monitoring the resource agent relies on interfaces

 provided by SAP. A third task of the resource agent is to also check the synchronisation status

@@ -205,9 +211,10 @@

         <longdesc lang="en">Should cluster/RA prefer to switchover to slave instance instead of restarting master locally? Default="yes"

         no: Do prefer restart locally

         yes: Do prefer takever to remote site

+        never: Do never run a sr_takeover (promote) at the secondary side. THIS VALUE IS CURRENTLY NOT SUPPORTED.

         </longdesc>

         <shortdesc lang="en">Local or site recover preferred?</shortdesc>

-        <content type="boolean" default="yes" />

+        <content type="string" default="yes" />

     </parameter>

     <parameter name="AUTOMATED_REGISTER"  unique="0" required="0">

         <shortdesc lang="en">Define, if a former primary should automatically be registered.</shortdesc>

@@ -220,7 +227,7 @@

         <shortdesc lang="en">Time difference needed between to primary time stamps, if a dual-primary situation occurs</shortdesc>

         <longdesc lang="en">Time difference needed between to primary time stamps,

         if a dual-primary situation occurs. If the time difference is

-        less than the time gap, then the cluster hold one or both instances in a "WAITING" status. This is to give an admin

+        less than the time gap, then the cluster holds one or both instances in a "WAITING" status. This is to give an admin

         a chance to react on a failover. A failed former primary will be registered after the time difference is passed. After

         this registration to the new primary all data will be overwritten by the system replication.

         </longdesc>

@@ -290,6 +297,45 @@

     local rc=0; tr -d '"'; return $rc

 }

+# function: version: cpmpare two HANA version strings

+function ver_lt() {

+    ocf_version_cmp $1 $2

+    test $? -eq 0 && return 0 || return 1

+}

+

+function ver_le() {

+    ocf_version_cmp $1 $2

+    test $? -eq 0 -o $? -eq 1 && return 0 || return 1

+}

+

+function ver_gt() {

+    ocf_version_cmp $1 $2

+    test $? -eq 2 && return 0 || return 1

+}

+

+function ver_ge() {

+    ocf_version_cmp $1 $2

+    test $? -eq 2 -o $? -eq 1 && return 0 || return 1

+}

+#

+# function: version: cpmpare two HANA version strings

+#

+function version() {

+    if [ $# -eq 3 ]; then

+        case "$2" in

+            LE | le | "<=" ) ver_le $1 $3;;

+            LT | lt | "<" ) ver_lt $1 $3;;

+            GE | ge | ">=" ) ver_ge $1 $3;;

+            GT | gt | ">" ) ver_gt $1 $3;;

+            * ) return 1;

+        esac

+    elif [ $# -ge 5 ]; then

+        version $1 $2 $3 && shift 2 && version $*

+    else

+        return 1;

+    fi

+}

+

 #

 # function: remoteHost2remoteNode - convert a SAP remoteHost to the cluster node name

 # params:   remoteHost

@@ -372,12 +418,16 @@

     dstr=$(date)

     case "$attr_store" in

         reboot | forever )

-            echo "$dstr: SAPHana: crm_attribute -N ${attr_node} -G -n \"$attr_name\" -l $attr_store -q" >> /var/log/fhATTRIBUTE

-            crm_attribute -N ${attr_node} -G -n "$attr_name" -l $attr_store -q -d "$attr_default" 2>>/var/log/fhATTRIBUTE; rc=$?

+            if ocf_is_true "$log_attributes"; then

+                echo "$dstr: SAPHana: crm_attribute -N ${attr_node} -G -n \"$attr_name\" -l $attr_store -q" >> $log_attr_file

+            fi

+            crm_attribute -N ${attr_node} -G -n "$attr_name" -l $attr_store -q -d "$attr_default" 2>>$log_attr_file; rc=$?

             ;;

         props )

-            echo "$dstr: SAPHana: crm_attribute -G -n \"$attr_name\" -t crm_config -q" >> /var/log/fhATTRIBUTE

-            crm_attribute -G -n "$attr_name" -t crm_config -q -d "$attr_default" 2>>/var/log/fhATTRIBUTE; rc=$?

+            if ocf_is_true "$log_attributes"; then

+                echo "$dstr: SAPHana: crm_attribute -G -n \"$attr_name\" -t crm_config -q" >> $log_attr_file

+            fi

+            crm_attribute -G -n "$attr_name" -t crm_config -q -d "$attr_default" 2>>$log_attr_file; rc=$?

             ;;

     esac

     super_ocf_log info "FLOW $FUNCNAME rc=$rc"

@@ -405,12 +455,16 @@

         dstr=$(date)

         case "$attr_store" in

             reboot | forever )

-                echo "$dstr: SAPHana: crm_attribute -N $attr_node -v $attr_value -n \"$attr_name\" -l $attr_store" >> /var/log/fhATTRIBUTE

-                crm_attribute -N $attr_node -v $attr_value -n "$attr_name" -l $attr_store 2>>/var/log/fhATTRIBUTE; rc=$?

+                if ocf_is_true "$log_attributes"; then

+                    echo "$dstr: SAPHana: crm_attribute -N $attr_node -v $attr_value -n \"$attr_name\" -l $attr_store" >> $log_attr_file

+                fi

+                crm_attribute -N $attr_node -v $attr_value -n "$attr_name" -l $attr_store 2>>$log_attr_file; rc=$?

                 ;;

             props )

-                echo "$dstr: SAPHana: crm_attribute -v $attr_value -n \"$attr_name\" -t crm_config -s SAPHanaSR" >> /var/log/fhATTRIBUTE

-                crm_attribute -v $attr_value -n "$attr_name" -t crm_config  -s SAPHanaSR 2>>/var/log/fhATTRIBUTE; rc=$?

+                if ocf_is_true "$log_attributes"; then

+                    echo "$dstr: SAPHana: crm_attribute -v $attr_value -n \"$attr_name\" -t crm_config -s SAPHanaSR" >> $log_attr_file

+                fi

+                crm_attribute -v $attr_value -n "$attr_name" -t crm_config  -s SAPHanaSR 2>>$log_attr_file; rc=$?

                 ;;

         esac

     else

@@ -460,6 +514,10 @@

     # DONE: PRIO2: Only adjust master if value is really different (try to check that)

     oldscore=$(${HA_SBIN_DIR}/crm_master -G -q -l reboot)

     if [ "$oldscore" != "$score" ]; then

+       dstr=$(date)

+       if ocf_is_true "$log_attributes"; then

+           echo "$dstr: SAPHana: crm_master -v $score -l reboot " >> $log_attr_file

+       fi

        super_ocf_log debug "DBG: SET crm master: $score (old: $oldscore)"

        ${HA_SBIN_DIR}/crm_master -v $score -l reboot; rc=$?

     else

@@ -471,9 +529,9 @@

 }

 #

-# function: scoring_crm_master - score instance due to role ans sync match (table SCORING_TABLE_PREFERRED_SITE_TAKEOVER)

+# function: scoring_crm_master - score instance due to role ans sync match (table SCORING_TABLE)

 # params:   NODE_ROLES NODE_SYNC_STATUS

-# globals:  SCORING_TABLE_PREFERRED_SITE_TAKEOVER[@],

+# globals:  SCORING_TABLE[@],

 #

 scoring_crm_master()

 {

@@ -482,7 +540,7 @@

     local sync="$2"

     local skip=0

     local myScore=""

-    for scan in "${SCORING_TABLE_PREFERRED_SITE_TAKEOVER[@]}"; do

+    for scan in "${SCORING_TABLE[@]}"; do

         if [ $skip -eq 0 ]; then

             read rolePatt syncPatt score <<< $scan

             if grep "$rolePatt" <<< "$roles"; then

@@ -494,7 +552,7 @@

         fi

     done

     super_ocf_log debug "DBG: scoring_crm_master adjust score $myScore"

-    # TODO: PRIO1: DO Not Score, If we did not found our role/sync at this moment - bsc#919925

+    # DONE: PRIO1: DO Not Score, If we did not found our role/sync at this moment - bsc#919925

     if [ -n "$myScore" ]; then

         set_crm_master $myScore

     fi

@@ -514,28 +572,91 @@

 }

 #

+# function: HANA_CALL

+# params:   timeout-in-seconds cmd-line

+# globals:  sid(r), SID(r), InstanceName(r)

+#

+function HANA_CALL()

+{

+    #

+    # TODO: PRIO 5: remove 'su - ${sidadm} later, when SAP HANA resoled issue with

+    #       root-user-called hdbnsutil -sr_state (which creates root-owned shared memory file in /var/lib/hdb/SID/shmgrp)

+    # TODO: PRIO 5: Maybe make "su" optional by a parameter

+    local timeOut=0

+    local onTimeOut=""

+    local rc=0

+    local use_su=1 # Default to be changed later (see TODO above)

+    local pre_cmd=""

+    local cmd=""

+    local pre_script=""

+    local output=""

+    while [ $# -gt 0 ]; do

+        case "$1" in

+            --timeout ) timeOut=$2; shift;;

+            --use-su  ) use_su=1;;

+            --on-timeout ) onTimeOut="$2"; shift;;

+            --cmd ) shift; cmd="$*"; break;;

+        esac

+        shift

+    done

+

+    if [ $use_su -eq 1 ]; then

+        pre_cmd="su - ${sid}adm -c"

+        pre_script="true"

+    else

+        # as root user we need the library path to the SAP kernel to be able to call sapcontrol

+        # check, if we already added DIR_EXECUTABLE at the beginning of LD_LIBRARY_PATH

+        if [ "${LD_LIBRARY_PATH%%*:}" != "$DIR_EXECUTABLE" ]

+        then

+            MY_LD_LIBRARY_PATH=$DIR_EXECUTABLE${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH

+        fi

+        pre_cmd="bash -c"

+        pre_script="LD_LIBRARY_PATH=$MY_LD_LIBRARY_PATH; export LD_LIBRARY_PATH"

+    fi

+    case $timeOut in

+        0 | inf )

+                  output=$($pre_cmd "$pre_script; /usr/sap/$SID/$InstanceName/HDBSettings.sh $cmd"); rc=$?

+                  ;;

+        *       )

+                  output=$(timeout $timeOut $pre_cmd "$pre_script; /usr/sap/$SID/$InstanceName/HDBSettings.sh $cmd"); rc=$?

+                  #

+                  # on timeout ...

+                  #

+                  if [ $rc -eq 124 -a -n "$onTimeOut" ]; then

+                      local second_output=""

+                      second_output=$($pre_cmd "$pre_script; /usr/sap/$SID/$InstanceName/HDBSettings.sh $onTimeOut");

+                  fi

+                  ;;

+    esac

+    echo "$output"

+    return $rc;

+}

+

+#

 # function: saphana_init - initialize variables for the resource agent

 # params:   InstanceName

-# globals:  OCF_*(r), SID(w), sid(rw), sidadm(w), InstanceName(w), InstanceNr(w), SAPVIRHOST(w), PreferSiteTakeover(w), 

-# globals:  sr_name(w), remoteHost(w), otherNodes(w), rem_SR_name(w)

+# globals:  OCF_*(r), SID(w), sid(rw), sidadm(w), InstanceName(w), InstanceNr(w), SAPVIRHOST(w), PreferSiteTakeover(w),

+# globals:  sr_name(w), remoteHost(w), otherNodes(w), remSR_name(w)

 # globals:  ATTR_NAME_HANA_SYNC_STATUS(w), ATTR_NAME_HANA_CLONE_STATE(w)

 # globals:  DIR_EXECUTABLE(w), SAPSTARTSRV(w), SAPCONTROL(w), DIR_PROFILE(w), SAPSTARTPROFILE(w), LD_LIBRARY_PATH(w), PATH(w)

 # globals:  LPA_DIRECTORY(w), SIDInstanceName(w), remoteNode(w), hdbSrQueryTimeout(w)

+# globals:  NODENAME(w), vNAME(w), hdbver(w),

 # saphana_init : Define global variables with default values, if optional parameters are not set

 #

 function saphana_init() {

     super_ocf_log info "FLOW $FUNCNAME ($*)"

     local rc=$OCF_SUCCESS

-    local vName

     local clN

     # local site

     # two parameter models (for transition only)

     # OLD: InstanceName

     # NEW: SID InstanceNumber

+    NODENAME=$(crm_node -n)

     SID=$OCF_RESKEY_SID

     InstanceNr=$OCF_RESKEY_InstanceNumber

     SIDInstanceName="${SID}_HDB${InstanceNr}"

     InstanceName="HDB${InstanceNr}"

+    export SAPSYSTEMNAME=$SID

     super_ocf_log debug "DBG: Used new method to get SID ($SID) and InstanceNr ($InstanceNr)"

     sid=$(echo "$SID" | tr [:upper:] [:lower:])

     sidadm="${sid}adm"

@@ -544,15 +665,23 @@

     # DONE: PRIO4: SAPVIRHOST might be different to NODENAME

     # DONE: PRIO1: ASK: Is the output format of ListInstances fix? Could we take that as an API? Answer: Yes

     # try to catch:  Inst Info : LNX - 42 - lv9041 - 740, patch 36, changelist 1444691

-    # We rely on the following format: SID is word#4, NR is work#6, vHost is word#8

-    vName=$(/usr/sap/hostctrl/exe/saphostctrl -function ListInstances \

-        | awk '$4 == SID && $6=NR { print $8 }' SID=$SID NR=$InstanceNr)

+    # We rely on the following format: SID is word#4, SYSNR is work#6, vHost is word#8

+    if [ -e /usr/sap/hostctrl/exe/saphostctrl ]; then

+        vName=$(/usr/sap/hostctrl/exe/saphostctrl -function ListInstances \

+            | awk '$4 == SID && $6 == SYSNR { print $8 }' SID=$SID SYSNR=$InstanceNr 2>/dev/null )

+        super_ocf_log debug "DBG: ListInstances: $(/usr/sap/hostctrl/exe/saphostctrl -function ListInstances)"

+    else

+        super_ocf_log error "ERR: SAPHOSTAGENT is not installed at /usr/sap/hostctrl/exe (saphostctrl missing)"

+    fi

     if [ -z "$vName" ]; then

        #

        # if saphostctrl does not know the answer, try to fallback to attribute provided by SAPHanaTopology

        #

        vName=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_VHOST[@]} "$NODENAME");

     fi

+    if [ -z "$vName" ]; then # last fallback if we are not able to figure out the virtual host name

+       vName="$NODENAME"

+    fi

     SAPVIRHOST=${vName}

     PreferSiteTakeover="$OCF_RESKEY_PREFER_SITE_TAKEOVER"

     AUTOMATED_REGISTER="${OCF_RESKEY_AUTOMATED_REGISTER:-false}"

@@ -571,6 +700,12 @@

     ATTR_NAME_HANA_SRMODE=("hana_${sid}_srmode" "forever")

     ATTR_NAME_HANA_VHOST=("hana_${sid}_vhost" "forever")

     ATTR_NAME_HANA_STATUS=("hana_${sid}_status" "reboot")

+    ATTR_NAME_HANA_OPERATION_MODE=("hana_${sid}_op_mode" "forever")

+    #

+    # new "central" attributes

+    #

+    ATTR_NAME_HANA_FILTER=("hana_${sid}_glob_filter" "props" "ra-act-dec-lpa")

+    SAPHanaFilter=$(get_hana_attribute "X" ${ATTR_NAME_HANA_FILTER[@]})

     #

     # TODO: PRIO4: Table for non-preferred-site-takeover

     #

@@ -591,9 +726,7 @@

     )

     SCORING_TABLE_PREFERRED_LOCAL_RESTART=(

        "[0-9]*:P:[^:]*:master .*  150"

-       "[0-9]*:P:[^:]*:slave  .*  140"

-       "[0-9]*:P:[^:]*:\?     .*    0"

-       "[0-9]*:P:[^:]*:-     .*     0"

+       "[0-9]*:P:[^:]*:.*     .*  140"

        "[0-9]*:S:[^:]*:master SOK    100"

        "[0-9]*:S:[^:]*:master SFAIL  -INFINITY"

        "[0-9]*:S:[^:]*:slave  SOK     10"

@@ -602,6 +735,25 @@

        "[0-9]*:S:[^:]*:-      .*    0"

        ".*                    .*   -1"

     )

+    SCORING_TABLE_PREFERRED_NEVER=(

+       "[234]*:P:[^:]*:master .*     150"

+       "[015-9]*:P:[^:]*:master .*    90"

+       "[0-9]*:P:[^:]*:.*     .*     -INFINITY"

+       "[0-9]*:S:[^:]*:.*     .*     -INFINITY"

+       ".*                    .*     -INFINITY"

+    )

+    if ocf_is_true $PreferSiteTakeover; then

+        SCORING_TABLE=("${SCORING_TABLE_PREFERRED_SITE_TAKEOVER[@]}")

+    else

+        case "$PreferSiteTakeover" in

+            never|NEVER|Never )

+                SCORING_TABLE=("${SCORING_TABLE_PREFERRED_NEVER[@]}")

+                ;;

+            * )

+                SCORING_TABLE=("${SCORING_TABLE_PREFERRED_LOCAL_RESTART[@]}")

+                ;;

+        esac

+    fi

     #

     DUPLICATE_PRIMARY_TIMEOUT="${OCF_RESKEY_DUPLICATE_PRIMARY_TIMEOUT:-7200}"

     super_ocf_log debug "DBG: DUPLICATE_PRIMARY_TIMEOUT=$DUPLICATE_PRIMARY_TIMEOUT"

@@ -615,7 +767,7 @@

     esac

     #

     #

-

+    #

     remoteHost=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_REMOTEHOST[@]});

     if [ -z "$remoteHost" ]; then

        if [ ${#otherNodes[@]} -eq 1 ]; then # we are a 2 node cluster, lets assume the other is the remote-host

@@ -640,7 +792,7 @@

         sr_mode="sync"

     fi

     if [ -n "$remoteNode" ]; then

-        rem_SR_name=$(get_hana_attribute ${remoteNode} ${ATTR_NAME_HANA_SITE[@]});

+        remSR_name=$(get_hana_attribute ${remoteNode} ${ATTR_NAME_HANA_SITE[@]});

     fi

     super_ocf_log debug "DBG: sr_name=$sr_name, remoteHost=$remoteHost, remoteNode=$remoteNode, sr_mode=$sr_mode"

     # optional OCF parameters, we try to guess which directories are correct

@@ -671,26 +823,21 @@

         #

         SAPSTARTPROFILE="$(ls -1 $DIR_PROFILE/${OCF_RESKEY_INSTANCE_PROFILE:-${SID}_${InstanceName}_*})"

     fi

-    # as root user we need the library path to the SAP kernel to be able to call sapcontrol

-    # check, if we already added DIR_EXECUTABLE at the beginning of LD_LIBRARY_PATH

-    if [ "${LD_LIBRARY_PATH%%*:}" != "$DIR_EXECUTABLE" ]

-    then

-        LD_LIBRARY_PATH=$DIR_EXECUTABLE${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH

-        export LD_LIBRARY_PATH

-    fi

     PATH=${PATH}:${DIR_EXECUTABLE}; export PATH

+    local ges_ver

+    ges_ver=$(HANA_CALL --timeout 10 --cmd "HDB version" | tr -d " " | awk -F: '$1 == "version" {print $2}')

+    hdbver=${ges_ver%.*.*}

+    #

+    # since rev 111.00 we should use a new hdbnsutil option to get the -sr_state

+    # since rev 112.03 the old option is changed and we should use -sr_stateConfiguration where ever possible

+    #

+    hdbState="hdbnsutil -sr_state"

+    hdbMap="hdbnsutil -sr_state"

+    if version "$hdbver" ">=" "1.00.111"; then

+        hdbState="hdbnsutil -sr_stateConfiguration"

+        hdbMap="hdbnsutil -sr_stateHostMapping"

+    fi

     super_ocf_log info "FLOW $FUNCNAME rc=$OCF_SUCCESS"

-    #############################

-    # TODO: PRIO9: To be able to call landscapeHostConfig.py without su (so as root)

-    # TODO: PRIO9: Research for environment script .htacces or something like that

-    #export SAPSYSTEMNAME=ZLF

-    #export DIR_INSTANCE=/usr/sap/ZLF/HDB02

-    #export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$DIR_INSTANCE/exe:$DIR_INSTANCE/exe/Python/lib

-    #export PYTHONPATH=$DIR_INSTANCE/$HOST:$DIR_INSTANCE/exe/python_support:$DIR_INSTANCE/exe

-    #export PYTHONHOME=$DIR_INSTANCE/exe/Python

-    #export SAP_RETRIEVAL_PATH=$DIR_INSTANCE/$HOST

-    #export DIR_EXECUTABLE=$DIR_INSTANCE/exe

-    #############################

     return $OCF_SUCCESS

 }

@@ -765,7 +912,11 @@

         # or ownership - they will be recreated by sapstartsrv during next start

         rm -f /tmp/.sapstream5${InstanceNr}13

         rm -f /tmp/.sapstream5${InstanceNr}14

-        $SAPSTARTSRV pf=$SAPSTARTPROFILE -D -u $sidadm

+        (

+          export PATH="$DIR_EXECUTABLE${PATH:+:}$PATH"

+          export LD_LIBRARY_PATH="$DIR_EXECUTABLE${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH"

+          $SAPSTARTSRV pf=$SAPSTARTPROFILE -D -u $sidadm

+        )

         # now make sure the daemon has been started and is able to respond

         local srvrc=1

         while [ $srvrc -eq 1 -a $(pgrep -f "sapstartsrv.*$runninginst" | wc -l) -gt 0 ]

@@ -809,31 +960,47 @@

 function check_for_primary() {

     super_ocf_log info "FLOW $FUNCNAME ($*)"

     local rc=$HANA_STATE_DEFECT

-    node_full_status=$(su - ${sidadm} -c "hdbnsutil -sr_state" 2>/dev/null )

-    node_status=$(echo "$node_full_status" | awk '$1=="mode:" {print $2}')

-    super_ocf_log debug "DBG: check_for_primary: node_status=$node_status"

-    # TODO: PRIO2: Maybe we need to use a fallback interface when hdbnsitil does not answer properly -> lookup in config files?

+    # TODO: PRIO 3: Check beginning from which SPS does SAP support HDBSettings.sh?

+    # TODO: Limit the runtime of hdbnsutil and use getParameter.py as fallback

+    # TODO: PRIO2: Maybe we need to use a fallback interface when hdbnsutil does not answer properly -> lookup in config files?

     #              This might also solve some problems when we could not figure-out the ilocal or remote site name

-    for i in  1 2 3 4 5 6 7 8 9; do

+    local chkMethod=""

+    for chkMethod in  hU hU hU gP; do

+       case "$chkMethod" in

+           gP )

+                local gpKeys=""

+                gpKeys=$(echo --key=global.ini/system_replication/{mode,site_name,site_id})

+                node_full_status=$(HANA_CALL --timeout 60 --cmd "HDBSettings.sh getParameter.py $gpKeys --sapcontrol=1" 2>&1 | awk -F/ 'BEGIN {out=0} /^SAPCONTROL-OK: <begin>/ { out=1 } /^SAPCONTROL-OK: <end>/ { out=0 } /=/ {if (out==1) {print $3} }')

+                node_status=$(echo "$node_full_status" | awk -F= '$1=="mode" {print $2}')

+                super_ocf_log info "ACT: Using getParameter.py as fallback - node_status=$node_status"

+                ;;

+           hU | * )

+                # DONE: PRIO1: Begginning from SAP HANA rev 112.03 -sr_state is not longer supported

+                node_full_status=$(HANA_CALL --timeout 60 --cmd "$hdbState" 2>/dev/null )

+                node_status=$(echo "$node_full_status" | awk '$1=="mode:" {print $2}')

+                super_ocf_log debug "DBG: check_for_primary: node_status=$node_status"

+                ;;

+       esac

        case "$node_status" in

            primary )

-               super_ocf_log info "FLOW: $FUNCNAME rc=HANA_STATE_PRIMARY"

-               return $HANA_STATE_PRIMARY;;

+               rc=$HANA_STATE_PRIMARY

+               break;;

            syncmem | sync | async )

-               super_ocf_log info "FLOW: $FUNCNAME rc=HANA_STATE_SECONDARY"

-               return $HANA_STATE_SECONDARY;;

+               rc=$HANA_STATE_SECONDARY

+               break;;

            none ) # have seen that mode on second side BEFEORE we registered it as replica

-               super_ocf_log info "FLOW: $FUNCNAME rc=HANA_STATE_STANDALONE"

-               return $HANA_STATE_STANDALONE;;

+               rc=$HANA_STATE_STANDALONE

+               break;;

            * )

                super_ocf_log err "ACT: check_for_primary:  we didn't expect node_status to be: <$node_status>"

                dump=$( echo $node_status | hexdump -C );

                super_ocf_log err "ACT: check_for_primary:  we didn't expect node_status to be: DUMP <$dump>"

-               node_full_status=$(su - ${sidadm} -c "hdbnsutil -sr_state" 2>/dev/null )

-               node_status=$(echo "$node_full_status" | awk '$1=="mode:" {print $2}')

+               # TODO: Limit the runtime of hdbnsutil and use getParameter.py as fallback

+               # SAP_CALL

                super_ocf_log debug "DEC: check_for_primary: loop=$i: node_status=$node_status"

                # TODO: PRIO1: Maybe we need to keep the old value for P/S/N, if hdbnsutil just crashes

        esac;

+       sleep 2

     done

     super_ocf_log info "FLOW $FUNCNAME rc=$rc"

     return $rc

@@ -854,12 +1021,18 @@

 {

     super_ocf_log info "FLOW $FUNCNAME ($*)"

     local rc=-1 srRc=0 all_nodes_other_side="" n="" siteParam=""

-    if [ -n "$rem_SR_name" ]; then

-       siteParam="--site=$rem_SR_name"

+    if [ -n "$remSR_name" ]; then

+       siteParam="--site=$remSR_name"

     fi

-    FULL_SR_STATUS=$(su - $sidadm -c "python $DIR_EXECUTABLE/python_support/systemReplicationStatus.py $siteParam" 2>/dev/null); srRc=$?

-    super_ocf_log info "DEC $FUNCNAME systemReplicationStatus.py (to site '$rem_SR_name')-> $srRc"

-    super_ocf_log info "FLOW $FUNCNAME systemReplicationStatus.py (to site '$rem_SR_name')-> $srRc"

+    # TODO: Get rid of the su by using a new interface:

+    # SAPSYSTEMNAME=SLE /usr/sap/SLE/HDB00/HDBSettings.sh systemReplicationStatus.py $siteParam

+    # TODO: Check beginning from which SPS does SAP support HDBSettings.sh?

+    # TODO: Limit the runtime of systemReplicationStatus.py

+    # SAP_CALL

+    # FULL_SR_STATUS=$(su - $sidadm -c "python $DIR_EXECUTABLE/python_support/systemReplicationStatus.py $siteParam" 2>/dev/null); srRc=$?

+    FULL_SR_STATUS=$(HANA_CALL --timeout 60 --cmd "systemReplicationStatus.py" 2>/dev/null); srRc=$?

+    super_ocf_log info "DEC $FUNCNAME systemReplicationStatus.py (to site '$remSR_name')-> $srRc"

+    super_ocf_log info "FLOW $FUNCNAME systemReplicationStatus.py (to site '$remSR_name')-> $srRc"

     #

     # TODO: PRIO2: Here we might also need to filter additional sites (if multi tier should be supported)

     #              And is the check for return code capable for chains?

@@ -890,7 +1063,7 @@

           # ok we should be careful and set secondary to SFAIL

           super_ocf_log info "FLOW $FUNCNAME SFAIL"

           set_hana_attribute "$remoteNode" "SFAIL" ${ATTR_NAME_HANA_SYNC_STATUS[@]}

-          super_ocf_log info "ACT site=$sr_name, seting SFAIL for secondary (5) - srRc=$srRc lss=$lss"

+          super_ocf_log info "ACT site=$sr_name, setting SFAIL for secondary (5) - srRc=$srRc lss=$lss"

           # TODO: PRIO1 - P004: need to check LSS again to avoid dying primary to block (SFAIL) secondary

           lpa_set_lpt 10 "$remoteNode"

           rc=1

@@ -898,7 +1071,7 @@

     else

        super_ocf_log info "FLOW $FUNCNAME SFAIL"

        set_hana_attribute "$remoteNode" "SFAIL" ${ATTR_NAME_HANA_SYNC_STATUS[@]}

-       super_ocf_log info "ACT site=$sr_name, seting SFAIL for secondary (2) - srRc=$srRc"

+       super_ocf_log info "ACT site=$sr_name, setting SFAIL for secondary (2) - srRc=$srRc"

        # TODO: PRIO1 - P004: need to check LSS again to avoid dying primary to block (SFAIL) secondary

        lpa_set_lpt 10 "$remoteNode"

        rc=1;

@@ -992,14 +1165,28 @@

     super_ocf_log info "FLOW $FUNCNAME ($*)"

     local rc=0

     #

-    su - $sidadm -c "python $DIR_EXECUTABLE/python_support/landscapeHostConfiguration.py" 1>/dev/null 2>/dev/null; rc=$?

+    # TODO: Get rid of the su by using a new interface:

+    # SAPSYSTEMNAME=SLE /usr/sap/SLE/HDB00/HDBSettings.sh landscapeHostConfiguration.py

+    # TODO: Check beginning from which SPS does SAP support HDBSettings.sh?

+    # DONE: Limit the runtime of landscapeHostConfiguration.py

+    HANA_CALL --timeout 60 --cmd "landscapeHostConfiguration.py" 1>/dev/null 2>/dev/null; rc=$?

+    if [ $rc -eq 124 ]; then

+        # TODO: PRIO 1: Check, if we should loop here like 'for i in 1 2 3 ...' ?

+        # landscape timeout

+        sleep 20

+        HANA_CALL --timeout 60 --cmd "landscapeHostConfiguration.py" 1>/dev/null 2>/dev/null; rc=$?

+        if [ $rc -eq 124 ]; then

+           # TODO PRIO2: How to handle still hanging lss - current solution is to say "FATAL"

+           rc=0

+        fi

+    fi

     return $rc;

 }

 #

 # function: register_hana_secondary - register local hana as secondary to the other site

 # params:   -

-# globals:  sidadm(r), remoteHost(r), InstanceNr(r), sr_mode(r), sr_name(r)

+# globals:  sidadm(r), remoteHost(r), InstanceNr(r), sr_mode(r), sr_name(r), hdbver(r)

 # register_hana_secondary

 #

 function register_hana_secondary()

@@ -1007,17 +1194,31 @@

     super_ocf_log info "FLOW $FUNCNAME ($*)"

     local rc=2;

     local remoteInstance="";

+    local newParameter=0

     remoteInstance=$InstanceNr

+

+

+    if version "$hdbver" ">=" "1.00.110"; then

+        newParameter=1

+    fi

+

     if ocf_is_true ${AUTOMATED_REGISTER}; then

-       #

-       #

-       #

-       #

-       #

-       super_ocf_log info "ACT: REGISTER: hdbnsutil -sr_register --remoteHost=$remoteHost --remoteInstance=$remoteInstance --mode=$sr_mode --name=$sr_name"

-       #

-       #

-       su - $sidadm -c "hdbnsutil -sr_register --remoteHost=$remoteHost --remoteInstance=$remoteInstance --mode=$sr_mode --name=$sr_name"; rc=$?

+       # TODO: Get rid of the su by using a new interface:

+       # SAPSYSTEMNAME=SLE /usr/sap/SLE/HDB00/HDBSettings.sh hdbnsutil -sr_register ...

+       # TODO: Check beginning from which SPS does SAP support HDBSettings.sh?

+       # TODO: Limit the runtime of hdbnsutil -sr_register ????

+       if [ $newParameter -eq 1 ]; then

+           local hanaOM=""

+           hanaOM=$(get_hana_attribute ${NODENAME} ${ATTR_NAME_HANA_OPERATION_MODE[@]})

+           if [ -n "$hanaOM" ]; then

+               hanaOM="--operationMode=$hanaOM"

+           fi

+           super_ocf_log info "ACT: REGISTER: hdbnsutil -sr_register --remoteHost=$remoteHost --remoteInstance=$remoteInstance --replicationMode=$sr_mode $hanaOM --name=$sr_name"

+           HANA_CALL --timeout inf --use-su --cmd "hdbnsutil -sr_register --remoteHost=$remoteHost --remoteInstance=$remoteInstance --replicationMode=$sr_mode $hanaOM --name=$sr_name"; rc=$?

+       else

+           super_ocf_log info "ACT: REGISTER: hdbnsutil -sr_register --remoteHost=$remoteHost --remoteInstance=$remoteInstance --mode=$sr_mode --name=$sr_name"

+           HANA_CALL --timeout inf --use-su --cmd "hdbnsutil -sr_register --remoteHost=$remoteHost --remoteInstance=$remoteInstance --mode=$sr_mode --name=$sr_name"; rc=$?

+       fi

        # backup_global_and_nameserver

     else

        super_ocf_log info "ACT: SAPHANA DROP REGISTER because AUTOMATED_REGISTER is set to FALSE"

@@ -1051,7 +1252,7 @@

     check_sapstartsrv

     rc=$?

     #

-    # TODO: ASK: PRIO5: For SCALE-OUT - do we need to use an other call like StartSystem? Or better to use the HDB command?

+    # DONE: ASK: PRIO5: For SCALE-OUT - do we need to use an other call like StartSystem? Or better to use the HDB command?

     #

     if [ $rc -eq $OCF_SUCCESS ]; then

         output=$($SAPCONTROL -nr $InstanceNr -function Start)

@@ -1169,7 +1370,7 @@

         0 ) # LPA says start-up

             lpa_advice="start"

             # TODO: PRIO1: We need to do a special handling for remote being a 234-Secondary in SR Status SOK

-            # if ( remote_role like [234]:S )  && ( remote_sync_status is SOK|PRIM ) && ( PreferSiteTakeover ) 

+            # if ( remote_role like [234]:S )  && ( remote_sync_status is SOK|PRIM ) && ( PreferSiteTakeover )

             #   then lpa_advice="wait"

             remoteRole=$(get_hana_attribute $remoteNode ${ATTR_NAME_HANA_ROLES[@]})

             remoteSync=$(get_hana_attribute $remoteNode ${ATTR_NAME_HANA_SYNC_STATUS[@]})

@@ -1193,17 +1394,20 @@

         1)  # LPA says register!

             lpa_advice="register"

             ;;

-        2)  # LPA says wait for second LPT

+        2)  # LPA says wait for older LPA to expire

+            lpa_advice="wait"

+            ;;

+        3) # LPA says to wait for remote LPA to be reported/announced

             lpa_advice="wait"

             ;;

-        3 | 4 ) # LPA says something is completely wrong - FAIL resource # TODO: PRIO1: RC3 for waiting remote side to report lss

+        4) # LPA says something is completely wrong - FAIL resource # TODO: PRIO1: RC3 for waiting remote side to report lss

             lpa_advice="fail"

             ;;

-        * ) # LPA failed with an unkonown status - FAIL resource

+        *) # LPA failed with an unkonown status - FAIL resource

             lpa_advice="fail"

             ;;

     esac

-    

+

     # DONE: PRIO2: Do we need to differ 0 and 1 here? While 0 is a fatal SAP error, 1 for down/error

     if [ $lss -eq 0 ]; then

        super_ocf_log err "ACT: get_hana_landscape_status reports FATAL"

@@ -1218,7 +1422,7 @@

                 2 | 3 | 4 ) # as landcape says we are up - just set the scores and return code

                     super_ocf_log info "LPA: landcape: UP, LPA: start ==> keep running"

                     LPTloc=$(date '+%s')

-                    lpa_set_lpt $LPTloc

+                    lpa_set_lpt $LPTloc $NODENAME

                     rc=$OCF_SUCCESS

                     ;;

                 1 ) # landcape says we are down, lets start and adjust scores and return code

@@ -1226,7 +1430,7 @@

                     saphana_start

                     rc=$?

                     LPTloc=$(date '+%s')

-                    lpa_set_lpt $LPTloc

+                    lpa_set_lpt $LPTloc $NODENAME

                     ;;

             esac

             scoring_crm_master "$my_role" "$my_sync"

@@ -1250,11 +1454,11 @@

                         if [ $primary_status -eq $HANA_STATE_SECONDARY ]; then

                             super_ocf_log info "ACT: Register successful"

                             lpa_push_lpt 10

-                            lpa_set_lpt  10

+                            lpa_set_lpt  10 $NODENAME

                             set_crm_master  0

                             saphana_start_secondary

                             rc=$?