Monitor the CRS Stack (Verbose)

Introduction

Provides verbose output of the RAC cluster using crsstat

detailed_crsstat.ksh

#!/bin/ksh

ESC_SEQ="\033["
NORM=$ESC_SEQ"0m"
RED=$ESC_SEQ"0;31m"
LRED=$ESC_SEQ"1;31m"
LGREEN=$ESC_SEQ"1;32m"

SDS_resource_list=${1:-$($GRID_HOME/bin/crsctl status resource |grep NAME |awk -F\= '{print $2}' |sed -e 's/ //g')}
SDS_node_list=$($GRID_HOME/bin/olsnodes)

if [[ $SDS_node_list = "" ]]
then SDS_node_list=$(hostname)
fi

SDS_node_list_no=$($GRID_HOME/bin/olsnodes|wc -l)

if [ $SDS_node_list_no -eq 0 ]
then SDS_node_list_no=1
fi

set +A SDS_resource_actual_state
set +A SDS_temp_array

#
#Print Headers
#

echo "=================================================================================================================================================================================="
printf "%-40s" ""

printf "%-40s" ""

for n in {1..$SDS_node_list_no}
do
  printf "%-30s" "NODE $n"
done

printf "%-7s" ""

echo ""

printf "%-40s" "RESOURCE"

printf "%-40s" "TARGET STATE"

for n in {1..$SDS_node_list_no}
do
  printf "%-30s" "ACTUAL STATE"
done

printf "%-6s" "STATUS"

echo ""
echo "=================================================================================================================================================================================="

for f in $SDS_resource_list
do

  #
  # Set-up variables
  #

  SDS_resourse=$f
  SDS_resource_target_no=$($GRID_HOME/bin/crsctl status resource $SDS_resourse |grep TARGET |awk -F\= '{print $2}' |awk -F\, '{print NF}')
  SDS_print_string="$SDS_resourse"
  SDS_temp_state=""
  SDS_general_error=0

  #
  #Set initial state of vaiables
  #

  for n in {1..$SDS_node_list_no}
  do
    SDS_resource_actual_state[$n]="N/A"
  done

  #
  #Set target state array for each resource
  #

  SDS_target_online_count=0
  SDS_target_offline_count=0
  for n in {1..$SDS_node_list_no}
  do
    SDS_temp_state=$($GRID_HOME/bin/crsctl status resource $SDS_resourse |grep TARGET |awk -F\= '{print $2}' |awk -v loopno="$n" -F\, '{print $loopno}'|sed -e 's/ //g')

    #
    #Increment count each time an ONLINE field is found
    #

    echo ${SDS_temp_state} |grep ONLINE >/dev/null

    if [ $? = 0 ]
    then ((SDS_target_online_count++))
    fi

    echo ${SDS_temp_state} |grep OFFLINE >/dev/null

    if [ $? = 0 ]
    then
      ((SDS_target_offline_count++))
      ((SDS_general_error++))
    fi

  done

  if [ $SDS_target_offline_count -eq 0 ]
  then
    SDS_resource_target_state="ONLINE on $SDS_target_online_count Nodes"
  else
    SDS_resource_target_state="ONLINE on $SDS_target_online_count Nodes and OFFLINE on $SDS_target_offline_count"
  fi

  #
  #Set actual state array for each resource
  #

  for n in {1..$SDS_resource_target_no}
  do
    SDS_state_count=0
    SDS_err=0
    SDS_temp_array[$n]=$($GRID_HOME/bin/crsctl status resource $SDS_resourse |grep STATE |awk -F\= '{print $2}' |awk -v loopno="$n" -F\, '{print $loopno}'|sed -e 's/^ //')

    for x in $SDS_node_list
    do

      ((SDS_state_count++))

      echo ${SDS_temp_array[$n]} |grep $x >/dev/null
      SDS_err=$?

      echo ${SDS_temp_array[$n]} |grep OFFLINE >/dev/null
      SDS_err=$(( $SDS_err + $? ))

      if [ $SDS_err -lt 2 ]
      then SDS_resource_actual_state[$SDS_state_count]=${SDS_temp_array[$n]}
      fi

      echo ${SDS_temp_array[$n]} |grep OFFLINE >/dev/null
      if [ $? = 0 ]
      then ((SDS_general_error++))
      fi

    done
  done

  #
  #Output results to screen
  #

  printf "%-40s" "$SDS_resourse"

  printf "%-40s" "${SDS_resource_target_state}"

  for n in {1..$SDS_node_list_no}
  do
    printf "%-30s" "${SDS_resource_actual_state[$n]}"
  done

  if [ $SDS_general_error -eq 0 ]
  then
    printf $LGREEN
    printf "%-6s" "GOOD"
    printf $NORM
  else
    printf $LRED
    printf "%-6s" "BAD"
    printf "$NORM"
  fi

  echo ""

done

Example Output

==================================================================================================================================================================================
                                                                                NODE 1                        NODE 2                        NODE 3
RESOURCE                                TARGET STATE                            ACTUAL STATE                  ACTUAL STATE                  ACTUAL STATE                  STATUS
==================================================================================================================================================================================
ora.DATA01.dg                           ONLINE on 3 Nodes                       ONLINE on mydbnode100       ONLINE on mydbnode200       ONLINE on mydbnode300       GOOD
ora.FRA01.dg                            ONLINE on 3 Nodes                       ONLINE on mydbnode100       ONLINE on mydbnode200       ONLINE on mydbnode300       GOOD
ora.LISTENER.lsnr                       ONLINE on 3 Nodes                       ONLINE on mydbnode100       ONLINE on mydbnode200       ONLINE on mydbnode300       GOOD
ora.LISTENER_SCAN1.lsnr                 ONLINE on 1 Nodes                       N/A                         ONLINE on mydbnode200       N/A                         GOOD
ora.LISTENER_SCAN2.lsnr                 ONLINE on 1 Nodes                       N/A                         N/A                         ONLINE on mydbnode300       GOOD
ora.LISTENER_SCAN3.lsnr                 ONLINE on 1 Nodes                       ONLINE on mydbnode100       N/A                         N/A                         GOOD
ora.MGMTLSNR                            ONLINE on 1 Nodes                       N/A                         ONLINE on mydbnode200       N/A                         GOOD
ora.REDO01.dg                           ONLINE on 3 Nodes                       ONLINE on mydbnode100       ONLINE on mydbnode200       ONLINE on mydbnode300       GOOD
ora.REDO02.dg                           ONLINE on 3 Nodes                       ONLINE on mydbnode100       ONLINE on mydbnode200       ONLINE on mydbnode300       GOOD
ora.VOTE01.dg                           ONLINE on 3 Nodes                       ONLINE on mydbnode100       ONLINE on mydbnode200       ONLINE on mydbnode300       GOOD
ora.asm                                 ONLINE on 3 Nodes                       ONLINE on mydbnode100       ONLINE on mydbnode200       ONLINE on mydbnode300       GOOD
ora.cvu                                 ONLINE on 1 Nodes                       ONLINE on mydbnode100       N/A                         N/A                         GOOD
ora.mgmtdb                              ONLINE on 1 Nodes                       N/A                         ONLINE on mydbnode200       N/A                         GOOD
ora.net1.network                        ONLINE on 3 Nodes                       ONLINE on mydbnode100       ONLINE on mydbnode200       ONLINE on mydbnode300       GOOD
ora.oc4j                                ONLINE on 1 Nodes                       ONLINE on mydbnode100       N/A                         N/A                         GOOD
ora.ons                                 ONLINE on 3 Nodes                       ONLINE on mydbnode100       ONLINE on mydbnode200       ONLINE on mydbnode300       GOOD
ora.scan1.vip                           ONLINE on 1 Nodes                       N/A                         ONLINE on mydbnode200       N/A                         GOOD
ora.scan2.vip                           ONLINE on 1 Nodes                       N/A                         N/A                         ONLINE on mydbnode300       GOOD
ora.scan3.vip                           ONLINE on 1 Nodes                       ONLINE on mydbnode100       N/A                         N/A                         GOOD
ora.mydbrc00.db                         ONLINE on 3 Nodes                       ONLINE on mydbnode100       ONLINE on mydbnode200       ONLINE on mydbnode300       GOOD
ora.mydbnode100.vip                     ONLINE on 1 Nodes                       ONLINE on mydbnode100       N/A                         N/A                         GOOD
ora.mydbnode200.vip                     ONLINE on 1 Nodes                       N/A                         ONLINE on mydbnode200       N/A                         GOOD
ora.mydbnode300.vip                     ONLINE on 1 Nodes                       N/A                         N/A                         ONLINE on mydbnode300       GOOD
Unless otherwise stated, the content of this page is licensed under Creative Commons Attribution-ShareAlike 3.0 License