#!/bin/bash

# ----------------------------------------------------------------
# file   : diag8k.sh
# purpose: Script for diagnostics
# author : Vittal.S
#
# usage  : diag8k
#
# modifications:
# 11-15-08  Initial Creation .
# ----------------------------------------------------------------

### Constants
_SELF_="$(basename "$(readlink -e "${BASH_SOURCE[0]}")" && echo X)" && \
readonly _SELF_="${_SELF_%$'\nX'}"
_SELFDIR_="$(dirname "$(readlink -e "${BASH_SOURCE[0]}")" && echo X)" && \
readonly _SELFDIR_="${_SELFDIR_%$'\nX'}"
readonly COMMONINC="${_SELFDIR_}/common.inc"

### Include files
source ${COMMONINC} || exit 1

#Check if osv_version is available
OSV_VERSION=$(whereis -b osv_version.sh | awk '{ print $2 }')
if [ ! -f "$OSV_VERSION" ]; then
   logger --id $$ --priority user.warning "$0: osv_version.sh not found in \$PATH ($PATH), exiting!" >/dev/null 2>&1
   exit 1
fi

# ----------------------------------------------------------
# function: textncolor
# description:
#   sets up text in color.
# ----------------------------------------------------------

textncolor( )
{
   echo -e "${1}$2${Norm}"
}

setmode( )
{
   echo -e "$1"
}

pingtest( )
{
   var=$1
   ip=$2
   name=$3
   if [ "$ip" != "" ];
   then
      ping -q  -W 1 -c 1 $ip 1>/dev/null 2>&1
      status=$?
      eval ${var}stat=$status
      if [ $status -eq 0 ];
      then
         text="success"
      else
         text="failed"
      fi
   else
      eval ${var}stat=0
      text="not configured"
   fi
   echo "   - $name : $text"
}
   

checkcluster( )
{
   #<ConfiguredSAip>
   tmpsalog=/tmp/sa.log
   su - srx -c "RtpCfgShow -r Srx/Main/SurvivalAuthority 1>$tmpsalog 2>/dev/null" 2>/dev/null
   if [ $? -eq 0 ];
   then
      chmod 744 $tmpsalog
      survivalauth=`cat $tmpsalog 2>/dev/null | grep "^Value" | awk -F': ' '{ print $2 }' | sed -e 's/ //g'`
   else
      survivalauth=""
   fi
   rm $tmpsalog
   #</ConfiguredSAip>
   
   if osv_version.sh --major compare "$buildid" lt V5;
   then
      getcfgvar $CFGFILE node_1_cip_logical ciponfirst
      getcfgvar $CFGFILE node_2_cip_logical ciponsecond
      IPMICFG=/etc/opt/SMAW/SMAWsf/SA_ipmi.cfg
      SAIPMICMD=/usr/bin/SA_ipmi
   else
      ciponfirst="`cfgread nafo3 | awk '{ print $4 }'`"
      ciponsecond="`cfgread nafo3 | awk '{ print $5 }'`"
      IPMICFG=/etc/opt/SMAW/SMAWhaext/sa_ipmi.cfg
      SAIPMICMD=/opt/SMAW/SMAWhaext/bin/sa_ipmi
   fi

   #<DermineSAVariant>
   if osv_version.sh --major compare "$buildid" lt V4;
   then
      SACFG=/etc/opt/SMAW/SMAWsf/SA_die.cfg
      SACMD=/usr/bin/SA_die
   else
      if osv_version.sh --major compare "$buildid" lt V5;
      then
         SACFG=/etc/opt/SMAW/SMAWsf/SaQuery.cfg
         SACMD=/opt/SMAW/SMAWsf/bin/SA_down
	 comstrcfg="`cat $SACFG 2>/dev/null | grep -v '^#' | grep "community-string"            \
                                                        | awk '{ print $2 }' | sed -e 's/[ \t]*//g'`"
	 rm $SACFG 2>/dev/null
         {
            echo "survival-authority ${survivalauth}"
            echo "community-string ${comstrcfg}"
            echo "cluster-name ${clustername}"
            echo "${firstnode}#${firstnodeip}"
            echo "${secondnode}#${secondnodeip}"
         } > $SACFG
      else
         SACMD=/opt/SMAW/SMAWhaext/bin/sa_down
         SACFG=/tmp/expected.cfg
	 rm $SACFG 2>/dev/null
         {
            echo "survival-authority ${survivalauth}"
            echo "community-string dynamic"
            echo "cluster-name ${clustername}"
            echo "${firstnode}#${firstnodeip}"
            echo "${secondnode}#${secondnodeip}"
         } > $SACFG
      fi
   fi
   #</DermineSAVariant>

   #<CheckNode>
      nodename=`uname -n`
      if [ "$nodename" = "$firstnode" ];
      then
         mynode=$firstnode
         myip=$firstnodeip
         othernode=$secondnode
         primary=true
      else
         mynode=$secondnode
         myip=$secondnodeip
         othernode=$firstnode
         primary=false
      fi
   #</CheckNode>

   rsaconfig=`cat $IPMICFG 2>/dev/null | grep "^[ ]*$mynode" | awk -F':' '{ print $1 }' | awk '{ print $2 }'`
   remrsaconfig=`cat $IPMICFG 2>/dev/null | grep "^[ ]*$othernode" | awk -F':' '{ print $1 }' | awk '{ print $2 }'`
   sacfg="`cat $SACFG 2>/dev/null | grep -v '^#' | grep "survival-authority"              \
                                                    | awk '{ print $2 }' | sed -e 's/[ \t]*//g'`"
   if osv_version.sh --major compare "$buildid" lt V5;
   then
      comstrcfg="`cat $SACFG 2>/dev/null | grep -v '^#' | grep "community-string"            \
                                                        | awk '{ print $2 }' | sed -e 's/[ \t]*//g'`"
   else
      comstrcfg=dynamic
   fi
   
   if [ "$testbed" = "cluster" ];
   then
      if osv_version.sh --major compare "$buildid" lt V5;
      then
         ipmicfg="`cat $IPMICFG  2>/dev/null | grep -v '^#' | head -1`"
         ssh -b $myip -o ConnectTimeout=8 $othernode "cat $SACFG 2>/dev/null | grep -v '^#' | grep \"survival-authority\""   \
                                             1>/tmp/remotesares 2>/dev/null
         ssh -b $myip -o ConnectTimeout=8 $othernode "cat $SACFG 2>/dev/null | grep -v '^#' | grep \"community-string\""     \
                                             1>/tmp/remotecomstrres 2>/dev/null
         remotesacfg=`cat /tmp/remotesares 2>/dev/null | awk '{ print $2 }' | sed -e 's/[ \t]*//g'`
         remotecomstrcfg=`cat /tmp/remotecomstrres 2>/dev/null | awk '{ print $2 }' | sed -e 's/[ \t]*//g'`
         rm -f /tmp/remotesares /tmp/remotecomstrres
      else
         remotesacfg=$sacfg
         remotecomstrcfg=dynamic
         ipmicfg=""
         grep -q sa_ipmi /etc/opt/SMAW/SMAWhaext/saDevSwitch.cfg
         if [ $? -eq 0 ];
         then
            ipmicfg="`cat $IPMICFG  2>/dev/null | grep -v '^#' | head -1`"
         fi
      fi
   fi
   
   textncolor $BoldCyan "Note: BMC reffered to below is your Board Management Controller."
   textncolor $BoldCyan "      This reffers to your RSA/IMM port."
   #[1]. Check the sacfg themselves
   n1_n1="   - No contact possible without correct configration."
   ipmicfgtext="  - Node#`uname -n` configured for BMC: success." 
   if [ "$sacfg" != "" ];
   then
      if [ "$survivalauthnodecfg" != "$sacfg" ];
      then
          textncolor $BoldCyan "Information : $mynode"
          echo "   - Community string: $comstrcfg"
          textncolor $BoldCyan "   - The survival authority on ($mynode)node.cfg ($survivalauthnodecfg) does not match one configured($sacfg)"
          textncolor $BoldCyan "   - May be it was configured after installation."
      fi
   else
      textncolor $BoldCyan "Information : $mynode"
      echo "   - Community string: $remotecomstrcfg"
      textncolor $BoldCyan "   - No survival authority configured($mynode)."
      textncolor $BoldRed  "   - Your failover will not work in some cases."
      textncolor $BoldRed  "   - Failure Scenario - when two nodes loose connectivity via imm and cluster interconnect."
      textncolor $BoldCyan "   - Check $SACFG for entry survival-authority."
      textncolor $BoldCyan "   - Also Check Srx/Main/SurvivalAuthority RTP parm."
   fi

   if [ "$rsaconfig" != "$remoteonfirst" -a "$remrsaconfig" != "$remoteonfirst" -a \
        "$rsaconfig" != "$remoteonsecond" -a "$remrsaconfig" != "$remoteonsecond" ];
   then
      textncolor $BoldCyan "   - BMC configuration in node.cfg does not match one used."
      if [ -f $IPMICFG ];
      then
         textncolor $BoldCyan "   - Not an error, maybe BMC was configured manully later?"
      else
         textncolor $BoldCyan "   - Not an error, maybe BMC was never configured?"
      fi
   fi
  
   echo

   if [ "$remotesacfg" != "" ];
   then
      if [ "$survivalauthnodecfg" != "$remotesacfg" ];
      then
          textncolor $BoldCyan "Information : $othernode"
          echo "   - Community string: $comstrcfg"
          textncolor $BoldCyan "   - The survival authority on ($othernode)node.cfg ($survivalauthnodecfg) does not match one configured($remotesacfg)"
          textncolor $BoldCyan "   - May be it was configured after installation."
      fi
   else
      textncolor $BoldCyan "Information : $othernode"
      echo "   - Community string: $remotecomstrcfg"
      textncolor $BoldCyan "   - No survival authority configured($othernode)."
      textncolor $BoldRed  "   - Your failover will not work in some cases."
      textncolor $BoldRed  "   - Failure Scenario - when two nodes loose connectivity via imm and cluster interconnect."
      textncolor $BoldCyan "   - Check $SACFG for entry survival-authority."
   fi

   echo

   if [ "$ipmicfg" != "" ];
   then
      textncolor $BoldCyan "diag8k: Good, rsa/imm configuration found."
      textncolor $BoldCyan "diag8k: Checking onboard rsa/imm configuration."
      $SAIPMICMD -s $mynode 2>/dev/null
      if [ $? -eq 0 ];
      then
         n1_n1="  - From:-$mynode/To:-$mynode: success."
      else
         n1_n1="  - From:-$mynode/To:-$mynode: failed."
      fi
      $SAIPMICMD -s $othernode 2>/dev/null
      if [ $? -eq 0 ];
      then
         n1_n2="  - From:-$mynode/To:-$othernode: success."
      else
         n1_n2="  - From:-$mynode/To:-$othernode: failed."
      fi
      ssh -b $myip -o ConnectTimeout=8 $othernode "$SAIPMICMD -s $mynode 2>/dev/null" 2>/dev/null
      if [ $? -eq 0 ];
      then
         n2_n1="  - From:-$othernode/To:-$mynode: success."
      else
         n2_n1="  - From:-$othernode/To:-$mynode: failed."
      fi
      ssh -b $myip -o ConnectTimeout=8 $othernode "$SAIPMICMD -s $othernode 2>/dev/null" 2>/dev/null
      if [ $? -eq 0 ];
      then
         n2_n2="  - From:-$othernode/To:-$othernode: success."
      else
         n2_n2="  - From:-$othernode/To:-$othernode: failed."
      fi
   else
      textncolor $BoldRed "diag8k: No, rsa/imm configuration found."
      textncolor $BoldRed "diag8k: Check $IPMICFG"
      ipmicfgtext="  - Node#`uname -n` not configured: failed." 
   fi

   echo

   textncolor $BoldCyan "[1].Configuration test."
   textncolor $BoldCyan "  This tests the configuration of `uname -n` for failover."
   echo "$ipmicfgtext"

   #[2]. Check the ip's themselves
   textncolor $BoldCyan "[2].Reachability test.[ icmp ]"
   textncolor $BoldCyan "  This tests the ping form `uname -n` for each of the interfaces below."
   textncolor $BoldCyan "  Note: Some routers block ping."
   textncolor $BoldCyan "  Hint: Check your network if this test fails."

   textncolor $BoldCyan "  - Node#1"
   pingtest remoteonfirst    "$remoteonfirst"      "BMC           "
   if [ "$primary" = "true" ];
   then
   pingtest remotegwonfirst  "$remotegwonfirst"    "Gateway       "
   fi
   pingtest ciponfirst       "$ciponfirst"         "Cluster Ip    "
   textncolor $BoldCyan "  - Node#2"
   pingtest remoteonsecond   "$remoteonsecond"     "BMC           "
   if [ "$primary" = "false" ];
   then
   pingtest remotegwonsecond "$remotegwonsecond"   "Gateway       "
   fi
   pingtest ciponsecond      "$ciponsecond"        "Cluster Ip    "
   textncolor $BoldCyan "  - Survival Authority"
   pingtest sacfg            "$sacfg"              "Survival Auth "

   #[3]. Check the security rules.
   textncolor $BoldCyan "[3].Checking the security rules [ iptables ]"
   textncolor $BoldCyan "  Hint: Check if the security rules to survival authority are present."
   if [ "$sacfg" != "" ];
   then
      echo $sacfg | grep -q ':'
      if [ $? -eq 0 ]
      then
         ipt=ip6tables
      else
         ipt=iptables
      fi
      securityrules="`$ipt -nL |  grep $sacfg |  sed -e s/^ACCEPT/'   ACCEPT'/g`"
      if [ "$securityrules" != "" ];
      then
         echo "$securityrules"
      else
         echo "  - No security rules were found for the configured Survival Authority ($sacfg)."
      fi
   else
      echo "   None."
   fi
   

   #[4]. Check the et contact with other node
   textncolor $BoldCyan "[4].Contact BMC test[ ipmi ]."
   textncolor $BoldCyan "  This tests the ipmi protocol from both nodes as below."
   textncolor $BoldCyan "  If this test fails, it means we are not able get an response for ipmi pdu from ipmi port."
   if [ "$ipmicfg" != "" ];
   then
      textncolor $BoldCyan "  Hint: Check User/Password in $IPMICFG if this test fails."
      echo "$n1_n1"
      echo "$n1_n2"
      echo "$n2_n1"
      echo "$n2_n2"
   else
      echo "$n1_n1"
   fi

   #[5]. Check SA. 
   if osv_version.sh --major compare "$buildid" lt V5;
   then
      $SACMD -s $mynode 1>/dev/null 2>/dev/null
   else
      $SACMD -s $othernode 1>/dev/null 2>/dev/null
   fi
   if [ $? -eq 0 ];
   then
         sa1="  - $mynode: success."
   else
         sa1="  - $mynode: failed."
   fi
   if osv_version.sh --major compare "$buildid" lt V5;
   then
      ssh -b $myip -o ConnectTimeout=8 $othernode "$SACMD -s $othernode 1>/dev/null 2>/dev/null" 2>/dev/null
   else
      ssh -b $myip -o ConnectTimeout=8 $othernode "$SACMD -s $mynode 1>/dev/null 2>/dev/null" 2>/dev/null
   fi
   if [ $? -eq 0 ];
   then
         sa2="  - $othernode: success."
   else
         sa2="  - $othernode: failed."
   fi
   textncolor $BoldCyan "[5].Contact Survival Authority test[ snmp ]."
   textncolor $BoldCyan "  Hint: Check $SACFG if this test fails."
   textncolor $BoldCyan "      : Also check the remote survival authority community string($comstrcfg)."
   echo "$sa1"
   echo "$sa2"
}

# ----------------------------------------------------------
# function: lookbusy
# description:
#   looks busy, appreciated sometimes too.
# ----------------------------------------------------------

lookbusy( )
{
   seconds=$1
   basesleep=".1"
   totalsleep=`echo "$basesleep 5" | awk '{ printf( "%.1f", $1 * $2 ) }'`
   count=`echo "$seconds $totalsleep" | awk '{ printf( "%d", $1 / $2 ) }'`
   while [ $count -gt 0 ];
   do
      echo -ne "-\ \r";  sleep $basesleep;
      echo -ne "-| \r";  sleep $basesleep;
      echo -ne "-/\r";   sleep $basesleep;
      echo -ne "--\r";   sleep $basesleep;
      echo -ne "--\r";   sleep $basesleep;
      echo -ne "\r"
      count=`echo $count 1 | awk '{ printf( "%d", $1 - $2 ) }'`
   done
}

initvars( )
{
   # --------------- Begin:Color Codes --------------
   BoldGreen="\033[1;32m"
   BoldBlue="\033[1;34m"
   BoldCyan="\033[1;36m"
   BoldRed="\033[1;31m"
   BoldWhite="\033[1;37m"
   UnderLine="\033[4m"
   Yellow="\033[1;33m"
   Norm="\033[0m"
   # --------------- End:Color Codes ----------------

   export EthDevices8kOnly=true
   CFGFILE=${1:-/etc/hiq8000/node.cfg}
   #</checkUserid>
   UidMisMatch=7
   USER=`id -u`
   if [ $USER -ne 0 ];
   then
       echo "You  must be root to run this script."
       exit $UidMisMatch
   fi
   #</checkUserid>
   
   FAILOVERTEMPLATE=${2:-/var/opt/image/failover.template}
   getcfgvar $CFGFILE node_1_name        firstnode
   getcfgvar $CFGFILE node_2_name        secondnode
   getcfgvar $CFGFILE node_1_ip          firstnodeip
   getcfgvar $CFGFILE node_2_ip          secondnodeip
   getcfgvar $CFGFILE hw_platform        hwplatform
   getcfgvar $CFGFILE srx_build_id       buildid
   getcfgvar $CFGFILE test_bed           testbed 
   getcfgvar $CFGFILE cluster_dev        clusterdev 
   getcfgvar $CFGFILE cluster_name       clustername 
   getcfgvar $CFGFILE rsa_1_ip           remoteonfirst 
   getcfgvar $CFGFILE rsa_2_ip           remoteonsecond 
   getcfgvar $CFGFILE rsa_1_gateway      remotegwonfirst 
   getcfgvar $CFGFILE rsa_2_gateway      remotegwonsecond 
   getcfgvar $CFGFILE survival_authority survivalauthnodecfg
         
}

initvars $*

if [ "$testbed" = "cluster" ];
then
   if osv_version.sh --major compare "$buildid" ge V6;
   then
      Fail=2
      textncolor $BoldCyan "Analyzing state of cluster."
      su - solid -c "RtpSolid -l" | egrep 'PRIMARY ACTIVE|SECONDARY ACTIVE'
      solidstatus=$?

      nodename=`uname -n`
      if [ "$nodename" = "$firstnode" ];
      then
         iptables-save | egrep -q "\-A OUTPUT \-d ${remoteonsecond}/32 \-j DROP|\-A OUTPUT \-d ${remotegwonsecond}/32 \-j DROP"
	  else
	     iptables-save | egrep -q "\-A OUTPUT \-d ${remoteonfirst}/32 \-j DROP|\-A OUTPUT \-d ${remotegwonfirst}/32 \-j DROP"
	  fi 2>/dev/null
      blockedimpi=$?
      
      if [ "$nodename" = "$firstnode" ];
      then
         ssh -b $nodename -o ConnectTimeout=8 $secondnode "
            iptables-save|egrep -q \"\-A OUTPUT \-d ${remoteonfirst}/32 \-j DROP|\-A OUTPUT \-d ${remotegwonfirst}/32 \-j DROP\"
         " 
      else
         ssh -b $nodename -o ConnectTimeout=8 $firstnode "
            iptables-save|egrep -q \"\-A OUTPUT \-d ${remoteonsecond}/32 \-j DROP|\-A OUTPUT \-d ${remotegwonsecond}/32 \-j DROP\"
         " 
      fi 2>/dev/null
      partnerblockedimpi=$?

      if [ $blockedimpi -eq 0 -o $partnerblockedimpi -eq 0 -o $solidstatus -ne 0 ];
      then
          echo "diag8k: Sorry, cluster is not fully up."
          echo "Hint: Bring cluster to state#4, on both nodes,  before running it again."
          echo "    : Check your iptables for rules blocking IPMI for partner."
          echo "Done."
          exit $Fail
      fi
   fi
   textncolor $BoldCyan "Analyzing for failover scenarios."
   /unisphere/srx3000/srx/bin/draw8k  $CFGFILE $FAILOVERTEMPLATE
   /unisphere/srx3000/srx/bin/secure8k
   lookbusy 8
   echo ""
   checkcluster
else
   textncolor $BoldCyan "diag8k: This test is valid only for a cluster."
fi
