#!/bin/sh

# -------------------------------------------------------
# file   : trace8k.sh
# purpose: Installation script to trace 8k diagnostics.
# author : Vittal.S
#
# usage  : trace8k 
#
# modifications:
# 03-05-08  Initial Creation .
# -------------------------------------------------------

#Check if osv_version is available
OSV_VERSION=$(whereis -b osv_version.sh | awk '{ print $2 }')
if [ ! -f "$OSV_VERSION" ]; then
   logger --id $$ --priority user.warning "$0: osv_version.sh not found in \$PATH ($PATH), exiting!" >/dev/null 2>&1
   exit 1
fi

# ---------------------------------------------------------
# function _mount 
# arg : none
#    description : a private function for outputing the
#    mount in tabbed format inlcuding bind mount points 
#
_mount( )
{

   if [ -h /etc/mtab ]
   then
      MOUNT_CMD="findmnt --fstab --evaluate"     
   else
      MOUNT_CMD="mount"
   fi 
   
   ret=`${MOUNT_CMD}`
   res=$?

   echo "$ret"
   return $res
   
}

# ----------------------------------------------------------
# function: textncolor
# description:
#   sets up text in color.
# ----------------------------------------------------------

textncolor( )
{
   echo -e "[*] ${1}$2${Norm}"
}

setmode( )
{
   echo -e $1
}

# ---------------------------------------------------------------
# function: clearstdin
# args    : None
# description:
#   clears stdin from previous associated data.
# ----------------------------------------------------------------

clearstdin( )
{
	# Only consume stdin if file descriptor 0 (stdin)
	# is open and refers to a terminal.
	if [ -t 0 ]
	then
		while read -r -t 1 -d ""
		do
			read -n 256 -r -s -d ""
		done
	fi
}

dumpsip( )
{
   option=$1
   file=${TRCDIR}/${2}
   if [ "$USER" = "root" ];
   then
      su - srx -c "echo \"sipsmdump:${file}\" > $file;sipsmdump $1 >> $file" 1>/dev/null 2>&1 
   else
      echo "sipsmdump:${file}" > $file;sipsmdump $1 >> $file 
   fi
}

xdmstatus( )
{
   if "$OSV_VERSION" --major compare sys ge V6;
   then
      textncolor $BoldCyan "Collecting Xdm status report."
      if [ "$USER" = "root" ];
      then
         su - srx -c "/unisphere/srx3000/callp/bin/XdmUnreg.exe -st > ${TRCDIR}/xdm-status" 1>/dev/null 2>&1
      else
         /unisphere/srx3000/callp/bin/XdmUnreg.exe -st > ${TRCDIR}/xdm-status
      fi
   fi
}

diskusage( )
{
   fs="/ /var /opt /unisphere /home /enterprise /software /global /tpa /tmp /cdr"
   echo "-------------------------------------------------------"
   echo "///////////////////////////////////////////////////////"
   echo "Complete disk usage"
   echo "-------------------"
   df -h -P 2>/dev/null
   echo "Did you know?The following directories occupy top space."
   echo "Check if you have placed any temporary files in one of these."
   for dridx in $fs
   do
      echo "   ${dridx}:"
      dirs="`du -x -S $dridx 2>/dev/null | sort -nr | head -5 | awk '{ print $2 }'`"
      for idx in $dirs
      do
         space=`du -s -h $idx  2>/dev/null| head -5 | awk '{ print $1 }'`
         echo "      - $idx( $space )"
      done
   done
   echo "-------------------------------------------------------"
   echo "///////////////////////////////////////////////////////"
} > ${TRCDIR}/usage

collect( )
{
   if [ "$DualNode" = "true" ];
   then
      checkremote
   else
      if [ "$testbed" = "cluster" ];
      then
         if [ "$mynode" != "$MasterNode" ];
         then
            checkremote "-master $MasterNode"
            return
         fi
      fi
   fi

   echo                 "///////////////////////////////////////////////"
   textncolor $BoldCyan "Current traced system : <$mynode>"
   echo                 "///////////////////////////////////////////////"

   #[1a]. Collect a days worth Rtp Dumplog.
   if [ "$mynode" = "$primhost" -o "$DisaterMode" = "true" ];
   then
      textncolor $BoldCyan "Collecting RtpDumpLog."
      if [ "$LimitedMode" = "true" ];
      then
         datestamp=`date -d yesterday +%y%m%d`
         if [ "$USER" = "root" ];
         then
            su - srx -c "RtpDumpLog -o ${TRCDIR}/RtpDumpLog.txt -s ${datestamp}0000" 1>/dev/null 2>&1
         else
            RtpDumpLog -o ${TRCDIR}/RtpDumpLog.txt -s ${datestamp}0000
         fi
         cat ${TRCDIR}/RtpDumpLog.txt | egrep -v "RtpAudMgr|RtpRecMgr" > ${TRCDIR}/RtpDumpLog.txt.limited
         if "$OSV_VERSION" --major compare sys lt V5;
         then
            cat ${TRCDIR}/RtpDumpLog.txt.limited |  \
            egrep -v "cron|RtpAdmCli"               \
            > ${TRCDIR}/RtpDumpLog.txt
         else
            cat ${TRCDIR}/RtpDumpLog.txt.limited |                               \
            egrep -v "cron|crm_mon|lrmd|crmd|pengine|tengine|stonithd|RtpAdmCli" \
            > ${TRCDIR}/RtpDumpLog.txt
         fi
         rm -f ${TRCDIR}/RtpDumpLog.txt.limited
      else
         datestamp=`date -d "last week" +%y%m%d`
         if [ "$USER" = "root" ];
         then
            su - srx -c "RtpDumpLog -o ${TRCDIR}/RtpDumpLog.txt -s ${datestamp}0000" 1>/dev/null 2>&1
         else
            RtpDumpLog -o ${TRCDIR}/RtpDumpLog.txt -s ${datestamp}0000
         fi
         if [ $? -ne 0 ];
         then
            RtpDumpLog -o ${TRCDIR}/RtpDumpLog.txt
         fi
      fi
   else
      textncolor $BoldCyan "Skipping RtpDumpLog."
   fi

   #[1b]. Checking RTT 
   textncolor $BoldCyan "Checking if Real Time Trace is active"
   /unisphere/srx3000/callp/bin/oprtt_ctl -lo 2>/dev/null | sed -n '2,$p' > ${TRCDIR}/rttflags
   /unisphere/srx3000/callp/bin/oprtt_startup_ctl 2>/dev/null | sed -n '3,$p' > ${TRCDIR}/startuprttflags
   if [ -s ${TRCDIR}/rttflags ];
   then
      textncolor $BoldRed   "    - real time tracing is active."
   else
      textncolor $BoldPurple "    - real time tracing is inactive."
   fi
   if [ -s ${TRCDIR}/startuprttflags ];
   then
      textncolor $BoldRed   "    - real time tracing on process startup is active."
   else
      textncolor $BoldPurple "    - real time tracing on process startup is inactive."
   fi
   
   #[1c]. Collect configuration files  
   collectcfg

   #[2a]. Collect cstasm/sipsm dump data
   textncolor $BoldCyan "Collecting sipdump information."
   textncolor $BoldPurple "    - summary"
   dumpsip "-L" "sipsmdump-summary"
   if [ "$QuickMode" = "false" ];
   then
      textncolor $BoldPurple "    - context contents"
      dumpsip "-c" "sipsmdump-context-contents"
      textncolor $BoldPurple "    - dialog contents"
      dumpsip "-d" "sipsmdump-dialog-contents"
      textncolor $BoldPurple "    - endpoint info contents"
      dumpsip "-i" "sipsmdump-endpoint-info-contents"
      textncolor $BoldPurple "    - endpoint host contents"
      dumpsip "-t" "sipsmdump-endpoint-host-contents"
      textncolor $BoldPurple "    - subscription contents" 
      dumpsip "-s" "sipsmdump-subscription-contents"
   fi

   xdmstatus
  
   file=${TRCDIR}/cstasmdump
   cclfile=${TRCDIR}/dispprocdump
   textncolor $BoldCyan "Collecting cstadump information."
   if [ "$USER" = "root" ];
   then
      su - srx -c "cstasmdump -f $file" 
      su - srx -c "dispProcDump -s -f $cclfile"
   else
      cstasmdump -f $file
      dispProcDump -a -f $cclfile 
   fi 1>/dev/null 2>&1


   #[2b]
   textncolor $BoldCyan "Collecting solid logs."
   {
      if [ "$LimitedMode" = "false" ];
      then
         cp  /var/RtpDb/solmsg.out     ${TRCDIR}
         cp  /var/RtpDb/solerror.out   ${TRCDIR}
         cp  /var/RtpDb/ssdebug*       ${TRCDIR}
         cp  /log/ss_debug_upgrade.tar ${TRCDIR}
         cp  /home/solid/RtpSolid.log0 ${TRCDIR}
         cat /home/solid/RtpSolid.log1     | grep -v "Demon State: READY" > ${TRCDIR}/RtpSolid.log1.errors
         cat /home/solid/RtpSolid.log1.bak | grep -v "Demon State: READY" > ${TRCDIR}/RtpSolid.log1.bak.errors
      else  
         syear=`date -d yesterday +%Y`
         smonth=`date -d yesterday +%m`
         sday=`date -d yesterday +%d`
         eyear=`date +%Y`
         emonth=`date +%m`
         eday=`date +%d`
         cat /var/RtpDb/solmsg.out | egrep "^${sday}.${smonth}|^${eday}.${emonth}" > ${TRCDIR}/solmsg.out
         cp  /var/RtpDb/solerror.out ${TRCDIR}
         cat /home/solid/RtpSolid.log0 |                                             \
         egrep "^[ \t]*${syear}/${smonth}/${sday}|^[ \t]*${eyear}/${emonth}/${eday}" \
         > ${TRCDIR}/RtpSolid.log0
         cat /home/solid/RtpSolid.log1 |                                             \
         grep -v "Demon State: READY"  |                                             \
         egrep "^[ \t]*${syear}/${smonth}/${sday}|^[ \t]*${eyear}/${emonth}/${eday}" \
         > ${TRCDIR}/RtpSolid.log1.errors
         cat /home/solid/RtpSolid.log1.bak |                                        \
         grep -v "Demon State: READY" |                                             \
         grep "^[ \t]*${syear}/${smonth}/${sday}|^[ \t]*${eyear}/${emonth}/${eday}" \
         > ${TRCDIR}/RtpSolid.log1.bak.errors
      fi
   } 2>/dev/null

   #[2c]
   textncolor $BoldCyan "Collecting licensing information." 
   {
      cp /opt/unisphere/srx3000/cla/trace/trace.txt ${TRCDIR}/clatrace.txt 
      cp /opt/unisphere/srx3000/cla/log/audit.log   ${TRCDIR}/claaudit.log
      cp /opt/unisphere/srx3000/cla/log/error.log   ${TRCDIR}/claerror.log
   } 2>/dev/null
   RtpAdmCli -p 40 -l sysad -x <<EOF > ${TRCDIR}/license.txt 2>/dev/null
licEndpointDisplay


exit
EOF
   cat   ${TRCDIR}/license.txt |             \
         awk  'BEGIN { copy = 0 }            \
               /License Usage/{ copy = 1 }   \
               /CLI>/{ copy = 0 }            \
                /Ok./{ copy = 0 }            \
                copy { print    }'           \
         > ${TRCDIR}/license
   rm -f ${TRCDIR}/license.txt

   #[2d]
   if [ -x /unisphere/srx3000/callp/bin/XdmShmDisplay.exe ];
   then
      textncolor $BoldCyan "Collecting Xdm shared memory information." 
      /unisphere/srx3000/callp/bin/XdmShmDisplay.exe 1>/dev/null 2>&1
      cp /unisphere/srx3000/callp/etc/xdm/XdmShmInit.result.txt ${TRCDIR}
   fi

   #[2e]
   textncolor $BoldCyan "Collecting Customer RTP parms"
   if [ -f /opt/SMAW/SMAWrtp/cust_conf/zModifiedParms.parm ];
   then
      cp -p /opt/SMAW/SMAWrtp/cust_conf/zModifiedParms.parm ${TRCDIR}
      RtpCfgShow -o  ${TRCDIR}/rtpcfgshow.out 1>/dev/null 2>&1
   fi

   #[3]. Collect translation data.
   if [ "$LimitedMode" = "false" ];
   then
      textncolor $BoldCyan "Collecting translation information."
      echo -e "$BoldPurple"
      if [ "$QuickMode" = "false" ];
      then
         echo Y Y Y Y Y Y Y | /unisphere/srx3000/callp/bin/xlaMemDisplay 2>/dev/null \
                            | grep -v '?' | grep -v "XLA MEM"                        \
                            | grep -v "srx on "                                      \
                            | grep -v "^[ \t]*$"                                     \
                            | sed -e 's/^ /[*]     - /g'
      else
         echo N Y N N N N N | /unisphere/srx3000/callp/bin/xlaMemDisplay 2>/dev/null \
                            | grep -v '?' | grep -v "XLA MEM"                        \
                            | grep -v "srx on "                                      \
                            | grep -v "^[ \t]*$"                                     \
                            | sed -e 's/^ /[*]     - /g'
      fi
      echo -e "$Norm"
      mv -f /unisphere/srx3000/callp/etc/xla/xlaInit.result.txt $TRCDIR
   fi

   textncolor $BoldCyan "Collecting display number modification information."
   if [ -x /unisphere/srx3000/callp/bin/ndalMemDisplay ];
   then
      echo "Y Y Y Y Y" | /unisphere/srx3000/callp/bin/ndalMemDisplay 
      cp /unisphere/srx3000/callp/etc/ndal/ndalInit.result.txt ${TRCDIR} 
   fi 1>/dev/null 2>&1
    
   textncolor $BoldCyan "Collecting mlhg information."
   if [ -x /unisphere/srx3000/callp/bin/mlhgprint ];
   then
   {
      pushd /tmp
      /unisphere/srx3000/callp/bin/mlhgprint 
      cp /tmp/mlhgSharedMemory.txt ${TRCDIR}
      popd
   }
   fi 1>/dev/null 2>&1

   #[4a]. Collect the RSS size for various processes which are started by node manager
   textncolor $BoldCyan "Collecting memory utilization."
   fatherpid="`ps -o  pid,command ax| grep -v grep | grep RtpNm | awk '{ print $1 }'`"
   {
      echo " pid : command : vsz : rss : size" 
      echo " pid     : pid of process being analyzed"
      echo " command : command that was executed when the process was started"
      echo " vsz     : total virtual memory size"
      echo " rss     : resident memory size"
      echo " sz      : total size of writable pages for the process."
   } > ${TRCDIR}/rssinfo
   for idx in $fatherpid
   do
      ps -o  pid,command,vsz,rss,size ax  | egrep '^[ ]*$idx' >>  ${TRCDIR}/rssinfo
      ps -o  ppid,pid,command ax | grep "$idx" | awk '{ print $2 }' >> ${TRCDIR}/allprocs  2>/dev/null
   done
   for idx in `cat ${TRCDIR}/allprocs 2>/dev/null`
   do
       ps -o  pid,command,vsz,rss,size ax  | egrep "^[ ]*$idx" 
   done  >> ${TRCDIR}/rssinfo
   rm -f ${TRCDIR}/allprocs

   textncolor $BoldCyan "Collecting system information"
   {
      echo "Free memory"
      echo "-----------"
      free -m
   } > ${TRCDIR}/freemem


   {
      echo "Process status"
      echo "--------------"
      cat  /proc/*/status 2>/dev/null 
      cat  /proc/meminfo
      echo "-------  process tree ---------"
      ps -efH
   } > ${TRCDIR}/procstatus

   {
      echo "Top info"
      echo "--------"
      top -b -n 2
   }  > ${TRCDIR}/topinfo

   textncolor $BoldCyan "Collecting ipcs information."
   {
      echo "ipcs info(ipcs -a)"
      echo "------------------"
      ipcs -a
   }  > ${TRCDIR}/ipcs
   
   textncolor $BoldCyan "Collecting /software listing."
   {
      echo "/software listing (ls -ltR)"
      echo "---------------------------"
      ls -ltR /software --hide=export 2>/dev/null
      echo "/software usage( du /software -h)"
      du /software  -h 2>/dev/null
   }  > ${TRCDIR}/software-listing

   textncolor $BoldCyan "Collecting dmesg"
   dmesg > ${TRCDIR}/dmesg

   textncolor $BoldCyan "Collecting disk usage"
   diskusage

   textncolor $BoldCyan "Collecting patchset information"
   {
      echo "pkgversion -ps"
      echo "--------------"
      pkgversion -ps
   } > ${TRCDIR}/patchlevel

   textncolor $BoldCyan "Collecting mop information"
   {
      echo "pkgversion -m"
      echo "-------------"
      pkgversion -m
   } >> ${TRCDIR}/patchlevel
   textncolor $BoldCyan "Collecting rpm packages"
   {
      echo "rpm -aq | sort"
      echo "--------------"
      rpm -aq | sort
   }  >> ${TRCDIR}/packages

   if [ -x /enterprise/assistant/scripts/versions.sh ];
   then
      textncolor $BoldCyan "Collecting applicaion versions"
      {
         /enterprise/assistant/scripts/versions.sh
      } > ${TRCDIR}/applicationversions
      textncolor $BoldCyan "Collecting applicaion logs"
      cp /log/tomcat.log       ${TRCDIR}
      cp /log/symphonia.log    ${TRCDIR}
      cp /log/usage.log        ${TRCDIR}
      cp /log/usage_tomcat.log ${TRCDIR}
   fi 2>/dev/null
   
   #[4b]. rapid stat info
   if [ "$QuickMode" = "false" ];
   then
      textncolor $BoldCyan   "Collecting rapid stat info."
      textncolor $BoldPurple "Enter root password of ${mynode} if prompted."
      if [ "$DualNode" = "true" -o "$mynode" = "$sechost" -o "$DisaterMode" = "true" ];
      then
         RapidOpt="-o"
      else
         RapidOpt="-b"
      fi
      if "$OSV_VERSION" --major compare sys ge V7;
      then
         RapidOpt="$RapidOpt -W"
      fi
      while ( true )
      do
         clearstdin
         su - root -c "                                                                   \
            rm -f /tmp/*.xfstab;                                                          \
            echo -e '${BoldCyan}';                                                        \
            echo '[*] Collecting secure8k information';                                   \
            /unisphere/srx3000/srx/bin/secure8k 1>${TRCDIR}/security 2>&1;                \
            echo '[*] Collecting draw8k information';                                     \
            /unisphere/srx3000/srx/bin/draw8k 1>${TRCDIR}/draw 2>&1;                      \
            echo '[*] Collecting diag8k information';                                     \
            /unisphere/srx3000/srx/bin/diag8k 1>${TRCDIR}/failover 2>&1;                  \
            echo '[-] RapidStat starting.';                                               \
            /unisphere/srx3000/srx/bin/RapidStat -R $RapidOpt 1>${TRCDIR}/rapidstat 2>/dev/null; \
            echo '[-] RapidStat done.';                                                   \
            echo '[*] Collecting system files.';                                          \
            {                                                                             \
              mkdir -p ${TRCDIR}/current;                                                 \
              cp -f /etc/hosts ${TRCDIR}/current;                                         \
              cp -f /etc/hosts.allow ${TRCDIR}/current;                                   \
              cp -f /etc/hosts.deny ${TRCDIR}/current;                                    \
              cp -f /etc/shadow ${TRCDIR}/current;                                        \
              cp -f /etc/resolv.conf ${TRCDIR}/current;                                   \
              cp -f /etc/ntp.conf ${TRCDIR}/current;                                      \
              cp -f /etc/ntp.conf.cluster ${TRCDIR}/current;                              \
              cp -f /etc/security/access.conf ${TRCDIR}/current;                          \
              cp -f /etc/hiq8000/routes.cfg ${TRCDIR}/current;                            \
              cp -f /etc/hiq8000/node.cfg ${TRCDIR}/current;                              \
              cp -f /etc/hiq8000/response.cfg ${TRCDIR}/current;                          \
              cp -R -f /etc/sysconfig/network ${TRCDIR}/current;                          \
              cp -R -f /etc/rules ${TRCDIR}/current;                                      \
              cp -R -f /usr/local/ssl ${TRCDIR}/current;                                  \
              cp -f /opt/SMAW/SMAWrtp/cust_conf/AltHiqAlarmFilters.tcn ${TRCDIR}/current; \
              cp -f /repository/upgrade8k-timing  ${TRCDIR};                              \
              cp -f /repository/upgrade8k-details ${TRCDIR};                              \
              cp -f /repository/upgrade8k-rescue ${TRCDIR};                               \
              cp -f /log/*oute* ${TRCDIR};                                                \
              cp -f /log/rapidstat.log ${TRCDIR};                                         \
            } 2>/dev/null;                                                                \
            showsel 1>${TRCDIR}/showsel.out 2>&1;                                         \
            echo '[*] Collecting boot messages.';                                         \
            {                                                                             \
              cp -f /var/log/boot.msg  ${TRCDIR};                                         \
              cp -f /var/log/boot.omsg ${TRCDIR};                                         \
              cp -f /var/log/ha-log*   ${TRCDIR};                                         \
              cp -f /var/log/ha-debug* ${TRCDIR};                                         \
              cp -f /var/log/RtpCsmd*  ${TRCDIR};                                         \
              cp -f /var/log/bonding*  ${TRCDIR};                                         \
              cp -f /var/opt/SMAWhaext/log/sa_ipmi.log  ${TRCDIR};                        \
              cp -f /var/opt/SMAWhaext/log/saDevSwitch.log ${TRCDIR};                     \
              cp -f /var/opt/SMAWhaext/log/sa_down.log  ${TRCDIR};                        \
              cp -f /var/log/racoon-statistics*.log ${TRCDIR};                            \
              find /img/varimg/$fallback/log/boot.msg -ctime -${Ndays}                    \
                   -exec cp {} ${TRCDIR}/boot.msg.fallback  2>/dev/null;                  \
              find /img/varimg/$fallback/log/boot.omsg -ctime -${Ndays}                   \
                   -exec cp {} ${TRCDIR}/boot.omsg.fallback 2>/dev/null;                  \
              cp -f /var/log/audit/audit.log ${TRCDIR};                                   \
              find /var/log -maxdepth 1 -mindepth 1 -name messages\*                      \
                   -ctime -${Ndays} 2>/dev/null  | while read file; do                    \
                 if [ -f \$file ];                                                        \
                 then                                                                     \
                   cp -f \$file ${TRCDIR} 2>/dev/null;                                    \
                 fi                                                                       \
              done;                                                                       \
              find /log/ovlData -maxdepth 1 -mindepth 1 -name \*                          \
                   -ctime -${Ndays} 2>/dev/null  | while read file; do                    \
                 if [ -f \$file ];                                                        \
                 then                                                                     \
                   cp -f \$file ${TRCDIR} 2>/dev/null;                                    \
                 fi                                                                       \
              done;                                                                       \
              find /var/log -maxdepth 1 -mindepth 1 -name boot\*                          \
                   -ctime -${Ndays} 2>/dev/null  | while read file; do                    \
                 if [ -f \$file ];                                                        \
                 then                                                                     \
                   cp -f \$file ${TRCDIR} 2>/dev/null;                                    \
                 fi                                                                       \
              done;                                                                       \
              chown -R srx ${TRCDIR};                                                     \
              chgrp -R rtpgrp ${TRCDIR};                                                  \
            } 2>/dev/null;                                                                \
            echo '[*] Collecting history.';                                               \
            {                                                                             \
               cp -f /root/.bash_history ${TRCDIR}/root.bash_history;                     \
               cp -f /unisphere/srx3000/srx/.kshrc_history ${TRCDIR}/srx.kshrc_history;   \
            } 2>/dev/null;                                                                \
            {                                                                             \
            find /repository/upgrade8k-timing -ctime -1  1>/dev/null 2>&1;                \
            if [ \$? -eq 0 ];                                                             \
            then                                                                          \
               echo '[*] Collecting fallback logs.';                                      \
               mkdir -p ${TRCDIR}/fallback;                                               \
               cp -f /img/rootimg/etc/hosts ${TRCDIR}/fallback;                           \
               cp -f /img/rootimg/etc/hosts.allow ${TRCDIR}/fallback;                     \
               cp -f /img/rootimg/etc/hosts.deny ${TRCDIR}/fallback;                      \
               cp -f /img/rootimg/etc/shadow ${TRCDIR}/fallback;                          \
               cp -f /img/rootimg/etc/resolv.conf ${TRCDIR}/fallback;                     \
               cp -f /img/rootimg/etc/ntp.conf ${TRCDIR}/fallback;                        \
               cp -f /img/rootimg/etc/ntp.conf.cluster ${TRCDIR}/fallback;                \
               cp -f /img/rootimg/etc/security/access.conf ${TRCDIR}/fallback;            \
               cp -f /img/rootimg/etc/hiq8000/routes.cfg ${TRCDIR}/fallback;              \
               cp -f /img/rootimg/etc/hiq8000/node.cfg ${TRCDIR}/fallback;                \
               cp -f /img/rootimg/etc/hiq8000/response.cfg ${TRCDIR}/fallback;            \
               cp -R -f /img/rootimg/etc/sysconfig/network ${TRCDIR}/fallback;            \      
               cp -R -f /img/rootimg/etc/rules ${TRCDIR}/fallback;                        \
               cp -R /img/rootimg/usr/local/ssl ${TRCDIR}/fallback;                       \
               cp -f /img/varimg/$fallback/RtpDb/solmsg.out     ${TRCDIR}/fallback;       \
               cp -f /img/varimg/$fallback/RtpDb/solerror.out   ${TRCDIR}/fallback;       \
               cp -f /img/homeimg/$fallback/solid/RtpSolid.log0  ${TRCDIR}/fallback;      \
               cp -f /img/homeimg/$fallback/solid/RtpSolid.log1  ${TRCDIR}/fallback;      \
               cp -f /img/homeimg/$fallback/solid/RtpSolid.log1.bak ${TRCDIR}/fallback;   \
               cp -f /img/optimg/$fallback/SMAW/SMAWrtp/cust_conf/AltHiqAlarmFilters.tcn ${TRCDIR}/fallback;  \
               cp -f /img/logimg/$fallback/ss_debug_upgrade.tar ${TRCDIR}/fallback;       \
               cp -f /img/varimg/$fallback/RtpDb/ssdebug* ${TRCDIR}/fallback;             \
            fi;                                                                           \
            } 2>/dev/null;                                                                \
            echo '[*] Collecting keep track information';                                 \
            {                                                                             \
              cp -f /var/log/keep_track.log ${TRCDIR};                                    \
              cp -f /etc/hiq8000/version    ${TRCDIR};                                    \
              chmod a+r ${TRCDIR}/keep_track.log;                                         \
            } 2>/dev/null;                                                                \
            echo '[*] Collecting netstat information';                                    \
            {                                                                             \
               echo 'netstat -anp';                                                       \
               echo '------------';                                                       \
               netstat -anp;                                                              \
            } > ${TRCDIR}/netstat;                                                        \
            echo '[*] Collecting arp information';                                        \
            {                                                                             \
               echo 'arp -vn';                                                            \
               echo '-------';                                                            \
               arp -vn;                                                                   \
            } > ${TRCDIR}/arp;                                                            \
            echo '[*] Collecting iptables information';                                   \
            {                                                                             \
               echo 'iptables -nL -v';                                                    \
               echo '------------';                                                       \
               iptables -nL -v;                                                           \
            } > ${TRCDIR}/iptables;                                                       \
            echo '[*] Collecting ifconfig information';                                   \
            {                                                                             \
               echo 'ifconfig -a';                                                        \
               echo '-----------';                                                        \
               ifconfig -a;                                                               \
            } > ${TRCDIR}/ifconfig;                                                       \
            echo '[*] Collecting route information';                                      \
            {                                                                             \
               echo 'route -n(static routes)';                                            \
               echo '-----------------------';                                            \
               route -n;                                                                  \
               echo 'ip route(source based routes)';                                      \
               echo '-----------------------------';                                      \
               ip route;                                                                  \
            } > ${TRCDIR}/routes;                                                         \
            echo '[*] Collecting process file descriptors.';                              \
            {                                                                             \
               ps -fu srx | awk '{print \" /proc/\"\$2\"/fd\"}' | xargs ls -l;            \
            } > ${TRCDIR}/procfd 2>/dev/null;                                             \
            echo '[*] Collecting hardware information';                                   \
            {                                                                             \
               echo 'hwinfo';                                                             \
               echo '------';                                                             \
               hwinfo;                                                                    \
            } > ${TRCDIR}/hwinfo 2>/dev/null;                                             \
            if "$OSV_VERSION" --major compare sys ge V5 && "$OSV_VERSION" --major compare sys lt V7; \
            then                                                                          \
               crm_mon -o1 > ${TRCDIR}/crmmon.out 2>/dev/null;                            \
            fi;                                                                           \
            echo -e '${Norm}';                                                            \
            chown -R srx.$group /software/data;                                           \
            exit 0                                                                        \
         "
         status=$?
         if [ $status -ne 0 ];
         then
             echo "trace8k: Sorry, may be wrong password was entered."
             while ( true )
             do
                clearstdin
                echo -n "trace8k: Would you like to try again (y/n) ? "
                read answer
                if [ "$answer" = "y" -o "$answer" = "n" ];
                then
                   break;
                fi
                continue
             done
             if [ "$answer" = "y" ];
             then
                continue;
             else
                echo "Aborted."
                break;
             fi
         else
            break;
         fi   
      done
   else
      textncolor $BoldCyan "Skipping rapid stat info."
   fi

   #[4b]. cpu usage
   textncolor $BoldCyan "Collecting cpu usage."
   mpstat -P ALL > ${TRCDIR}/cpuusage
   iostat >> ${TRCDIR}/cpuusage

   #[5]. collect logs
   textncolor $BoldCyan "Collecting osv logs that are week old."
   logs="                           \
       HiqLogAlarm.log              \
       HiqLogAlert.log              \
       HiqLogAlways.log             \
       HiqLogInfo.log               \
       HiqLogEvent.log              \
       HiqLogComClusterProblems.log \
       HiqLogSwError.log            \
       HiqLogLicensing.log          \
       HiqLogCallTrace.log          \
       HiqLogEndPoints.log          \
       HiqLogIpSec.log              \
       HiqLogOverload.log           \
       HiqLogProvisioning.log       \
       HiqLogTransactionTimeout.log \
       HiqLogSecurity.log           \
       HiqLogBlackList.log          \
       HiqLogVIP.log                \
       HiqLogEndPointProblems.log   \
       HiqLogBlackList.log          \
       TCGprov.log                  \
   " 
   for idx in $logs
   do
      textncolor $BoldPurple "    - Copying $idx"
      if [ "$LimitedMode" = "false" ];
      then
         cp /log/${idx}* ${TRCDIR}/. 2>/dev/null
      else
         find /log -name ${idx}\* -ctime -${Ndays} 2>/dev/null  | while read file; do
            if [ -f $file ];
            then
                  basefile="`basename $file`"
                  normday="`date -d yesterday +%d|sed -e 's/^0/ /'`"
                  tstamp="`date -d yesterday '+%a %b'` ${normday}"
                  linenum="`egrep -m 1 -n \"^time.*${tstamp}*\" $file | awk -F':' '{ print $1 }'`"
                  if [ "$linenum" != "" ];
                  then
                     if [ $linenum -gt 2 ];
                     then
                        linenum="`expr $linenum - 2`"
                        nlines="`cat $file| wc -l`"
                        nleft="`expr $nlines - $linenum + 1`"
                        tail -${nleft} $file > ${TRCDIR}/${basefile}
                     fi
                  fi
            fi
         done
      fi
   done
   
   textncolor $BoldCyan "Collecting rtp logs."
   find /unisphere/srx3000/srx/40/log -maxdepth 1 -mindepth 1 -name  \*.log   -ctime -${Ndays} -exec cp -p {} ${TRCDIR}/. \;
   find /unisphere/srx3000/srx/40/log -maxdepth 1 -mindepth 1 -name  \*.log.* -ctime -${Ndays} -exec cp -p {} ${TRCDIR}/. \;
   cp /log/ActiveAlarmList.xml ${TRCDIR}/. 2>/dev/null

   textncolor $BoldCyan "Collecting x-channel logs."
   {
      cp -p /var/log/opt/SMAWhaext/log/* ${TRCDIR} -exec cp -p {} ${TRCDIR}/.
      cp -p /unisphere/srx3000/callp/etc/cluster/* ${TRCDIR}/.
   } 2>/dev/null
   

   upgradelogs="                          \
      /img/logimg/primary/prepare8k.log   \
      /img/logimg/secondary/prepare8k.log \
   "
   {
      cp -p /img/logimg/primary/prepare8k.log ${TRCDIR}/prepare8k.log.primary
      cp -p /img/logimg/secondary/prepare8k.log ${TRCDIR}/prepare8k.log.secondary
   } 1>/dev/null 2>&1
  
   #[5]. Analyzing semaphores
   textncolor $BoldCyan "Checking semaphores."
   {
      echo "List of sems : - " 
      echo "-----------------" 
   } >> ${TRCDIR}/semstatus
   for idx in `ipcs -s | grep '^0xc0' | awk '{ print $1 }'`
   do
      key=$idx
      semid=`ipcs -s | grep $key | awk '{ print $2 }'`
      locked=`ipcs -si $semid | sed -n '9,$p' | awk '{ printf("%s ",$2) }'`
      lockedpid="`ipcs -si $semid | sed -n '9,$p' | awk '{ printf("%s ",$5) }' \
                                  |  tr ' ' '\n' | sort | uniq | paste -d' ' -s - | sed -e 's/[ ]*//'`"
      lastop=`ipcs -si $semid | grep '^otime' | awk -F'=' '{ print $2 }'`
      lastcr=`ipcs -si $semid | grep '^ctime' | awk -F'=' '{ print $2 }'`
      for lockedidx in $locked
      do
         if [ "$lockedidx" != "0" -a "$lockedidx" != "" ];
         then
            echo "    - $key : locked   : $lockedidx : $lastop : $lockedpid" >>  ${TRCDIR}/semstatus
            break;
         fi
      done
   done 
   {
      echo "Pids:"
      echo "-----"
   } >>${TRCDIR}/semstatus 

   #[6]. Analysing pstacks.
   textncolor $BoldCyan "Checking pstacks."
   mkdir ${TRCDIR}/pstacks
   {
      cp  -f -p /software/rtpcore/*pstack ${TRCDIR}/pstacks 
      cp  -f -p /software/rtpcore/*pmap   ${TRCDIR}/pstacks 
   } 2>/dev/null
   stklist="`ls -lrt *pstack 2>/dev/null | awk '{ print $9 }' | sed -e s/.[0-9]*.pstack//g | sort | uniq`"
   for idx in $stklist
   do
       textncolor $BoldPurple "    - $idx"
   done

   "${CREATE_SI_LIST}" -d ${TRCDIR} || echo "Failed to create static identifier list"

   pushd   $TRCROOT 1>/dev/null 2>&1
   tar -cf ${TRCDIR}.tar data

   if [ -x $OBF_TOOL ]; then
      textncolor $BoldCyan "Analyzing for PII"
      mkdir -p $OBF_DIR
      cp ${TRCDIR}.tar $OBF_DIR
      $OBF_TOOL -o $OBF_YAML $OBF_DIR
      [ -s $OBF_YAML ] &&
         tar rf ${TRCDIR}.tar -C $(dirname $OBF_YAML) $(basename $OBF_YAML)
   fi
   
   gzip -9 ${TRCDIR}.tar
   chown srx.$group ${TRCDIR}.tar.gz
   chmod ug+r ${TRCDIR}.tar.gz
   rm -rf $TRCDIR
   if [ "$LogFile" = "" ];
   then
      year=`date +%y`
      month=`date +%m`
      day=`date +%d`
      LogFile=/log/data-${mynode}-${month}-${day}-${year}.tar.gz
   fi
   mv ${TRCDIR}.tar.gz ${LogFile}
   popd 1>/dev/null 2>&1
   textncolor $BoldGreen "Locate the logs for $mynode at /log/data-${mynode}-${month}-${day}-${year}.tar.gz"
   textncolor $BoldCyan "Done."
} 

# -------------------------------------------------------
# function: checksys
# args    :  none
# description:
#   checks all critical errors.
# -------------------------------------------------------

checksys( ) {

   #<CheckHwAbstraction>
      cat /etc/fstab | grep "/img/homeimg" | grep -q "^/dev/sda"
      scsistat=$?
      if [ $scsistat -ne 0 ];
      then
         cat /etc/mtab | grep "/dev/sda14" | grep -q "var"
         scsistat=$?
      fi
      if [ $scsistat -eq 0 ];
      then
         echo "trace8k:Detected scsi disk."
         diskdev=sda
      else
         cat /etc/fstab | grep "/img/homeimg" | grep -q "^/dev/hda"
         if [ $? -eq 0 ];
         then
            echo "trace8k:Detected ide disk."
            diskdev=hda
         else
            echo "trace8k:Unkown disk type."
            exit $FAIL
         fi
      fi
   #</CheckHwAbstraction>
   grep -q primary /etc/mtab
   primstatus=$?
   grep -q  secondary /etc/mtab
   secstatus=$?
   if [ $primstatus -ne 0 -a $secstatus -ne 0 ];
   then
      bootdev="(hd0,1)"
      sdadev=${diskdev}2
   else
      if [ $primstatus -eq 0 ];
      then
         bootdev="(hd0,1)"
         sdadev=${diskdev}2
      else
         bootdev="(hd0,2)"
         sdadev=${diskdev}3
      fi
   fi
}

#---------------------------------------------------------------
# function: collectcfg 
# args    : None
# description:
#   collect cfg files
#---------------------------------------------------------------

collectcfg( )
{
   textncolor $BoldCyan "Collecting configuration files."
   cfgfile="                                  \
      /etc/hiq8000/node.cfg                   \
      /etc/snmp/agt/snmpd.cnf                 \
      /etc/opt/SMAW/SMAWsf/SA_die.cfg         \
      /etc/opt/SMAW/SMAWsf/SA_down.cfg        \
      /etc/opt/SMAW/SMAWsf/SaQuery.cfg        \
      /etc/opt/SMAW/SMAWsf/SA_ipmi.cfg        \
      /etc/opt/SMAW/SMAWsf/SA_snmp.cfg        \
      /etc/opt/SMAW/SMAWhaext/sa_down.cfg     \
      /etc/opt/SMAW/SMAWhaext.sa_ipmi.cfg     \
      /etc/opt/SMAW/SMAWhaext/saDevSwitch.cfg \
      /etc/opt/SMAW/SMAWhaext/sa_last.cfg     \
   "

   for idx in $cfgfile
   do
      if [ -r $idx ];
      then
         basecfgfile="`basename $idx`"
         cp $idx ${TRCDIR}/$basecfgfile
      fi
   done
}


#---------------------------------------------------------------
# function: checkremote
# args    : None
# description:
#   checks if remote node has the same level as current node.
# ---------------------------------------------------------------

checkremote( )
{
   remoteopt="$*"
   ps -ef f | grep asclient_reboot_done | egrep -v 'grep|startpar' 1>/dev/null 2>&1
   if [ $? -eq 0 ];
   then
      echo "trace8k: Should not use a system while the system has not finished intsall."
      echo "         : This is the case if asclient instance is running."
      echo "         : Will still continue."
   fi

   if [ "$testbed" != "cluster" ];
   then
      return
   fi


   if [ "$mynode" = "$primhost" ];
   then
      othernode=$sechost
      othercip=$seccip
      othercipname="clusternode2-priv"
      if [ "$QuickMode" = "true" ];
      then
         remoteopt="$remoteopt -quick"
      fi
      if [ "$QuietMode" = "true" ];
      then
         remoteopt="$remoteopt -quiet"
      fi
      if [ "$LogFile" != "" ];
      then
         remoteopt="$remoteopt -log ${LogFile}"
      fi
      if [ "$LimitedMode" = "true" ];
      then
         remoteopt="$remoteopt -limited"
      fi
      RemoteMode=true
      remoteopt="$remoteopt -remote"
   else
      othernode=$primhost
      othercip=$primcip
      othercipname="clusternode1-priv"
   fi

   cat /etc/fstab | grep "/home" | grep -v "^/dev/${diskdev}" | grep -q primary
   if [ $? -eq 0 ];
   then
      syncpart="primary"
      syncother="secondary"
   else
      syncpart="secondary"
      syncother="primary"
   fi
   rm -f /tmp/*.xfstab
   echo -n "Checking for mount<$syncpart> on partner node<${othernode}> : "
   #[1]. First try the cluster ip

   sshcip=$othercip
   ssh ${othercip} -to ConnectTimeout=8 "cat /etc/fstab 2>/dev/null" 1>/tmp/${othernode}.xfstab 2>/dev/null
   sshstatus=$?

   #[2]. Next try the logical name

   if [ $sshstatus -ne 0 ];
   then
      sshcip=$othercipname
      ssh ${othercipname} -to ConnectTimeout=8 "cat /etc/fstab 2>/dev/null" 1>/tmp/${othernode}.xfstab 2>/dev/null
      sshstatus=$?
   fi

   #[3]. Last try the hostname via admin network.

   if [ $sshstatus -ne 0 ];
   then
      sshcip=$othernode
      ssh ${othernode} -to ConnectTimeout=8 "cat /etc/fstab 2>/dev/null" 1>/tmp/${othernode}.xfstab 2>/dev/null
      sshstatus=$?
   fi

   if [ $sshstatus -eq 0 ];
   then
      grep -q $syncpart /tmp/${othernode}.xfstab 1>/dev/null 2>&1
      status=$?
      if [ $status -ne 0 ];
      then
         textncolor $BoldGreen "Interesting, Both the nodes are not installed on same <$syncpart> side."
         grep -q $syncother /tmp/${othernode}.xfstab 1>/dev/null 2>&1
         status=$?
      fi
      rm -f /tmp/*.xfstab
      if [ $status -eq 0 -o "$singleimaged" = "true" ];
      then
         echo "ok"
         if [ "$mynode" = "$primhost" ];
         then
            echo ""
            ssh ${sshcip} -to ConnectTimeout=8  "PATH=$PATH:/usr/local/bin $TRACE8k ${remoteopt}"
            if [ $? -ne 0 ];
            then
               echo "trace8k:Sorry, cannot $remoteopt remote node<$othernode>."
               exit $REMOTEOPTFAILED
            fi
            echo ""
         fi
      fi
   else
      echo "failed"
      if [ "$MasterNode" = "false" ];
      then
         exit $REMOTENODENOTALIVE
      else
         textncolor $BoldCyan "trace8k: Run trace8k on $othernode once its available."
      fi
   fi
   rm -f /tmp/*.xfstab
}

trapnexit( ) 
{
   rm -f /tmp/*.xfstab
   rm -rf $TRCDIR
   rm -rf $OBF_DIR
} 2>/dev/null

initvars( )
{
   stty intr undef 2>/dev/null
   TRCROOT=/software
   TRCDIR=${TRCROOT}/data
   OBF_TOOL='/unisphere/srx3000/srx/bin/gen_obf_lists.py'
   OBF_DIR=${TRCROOT}/obf
   OBF_YAML="${TRCDIR}/osv-regex-filespec.yaml"
   rm -rf $TRCDIR ${TRCDIR}.tar ${TRCDIR}-* /log/data-*.tar.gz
   mkdir -p $TRCDIR
   chmod 777 $TRCDIR

   BoldGreen="\033[1;32m"
   BoldBlue="\033[1;34m"
   BoldCyan="\033[1;36m"
   BoldWhite="\033[1;37m"
   BoldRed="\033[1;31m"
   BoldPurple="\033[1;35m"
   UnderLine="\033[4m"
   Yellow="\033[1;33m"
   Norm="\033[0m"

   SUCCESS=0
   FAIL=2
   DBCHECKFAILED=3
   REMOTEOPTFAILED=4
   REMOTENODENOTALIVE=5

   QuietMode=${QuietMode:-true}
   QuickMode=${QuickMode:-false}
   DualNode=${DualNode:-true}
   RemoteMode=${RemoteMode:-false}
   MasterNode=${MasterNode:-`uname -n`}
   DisaterMode=${DisaterMode:-false}
   LogFile=${LogFile:-""}
   Ndays=${Ndays:-28}
   trap trapnexit 0 1 2 3 9 15
   USER=`id -un`
   if [ "$USER" = "root" ];
   then
      export HOME=/unisphere/srx3000/srx
      source /unisphere/srx3000/srx/.profile
   else
      if [ "$USER" != "srx" ];
      then
         textncolor $BoldCyan "Rerun trace8k as user srx."
         exit 0
      fi
   fi

   grep -q rtpgrp /etc/group
   if [ $? -eq 0 ];
   then
      group=rtpgrp
   else
      group=other
   fi

   echo $0 | grep -q "/usr/bin/trace8k"
   if [ $? -eq 0 ];
   then
      TRACE8k=/usr/bin/trace8k
   else
      TRACE8k=/unisphere/srx3000/srx/bin/trace8k
   fi

   CREATE_SI_LIST=$(which create_si_list 2>/dev/null) || \
   CREATE_SI_LIST=/unisphere/srx3000/srx/bin/create_si_list
   [ -x ${CREATE_SI_LIST} ] || \
   echo "Error: ${CREATE_SI_LIST} not found! Unable to create static identifiers list."

   _mount | grep -q "/img/homeimg/secondary"
   if [ $? -eq 0 ];
   then
      fallback=primary
   else
      fallback=secondary
   fi
   mynode="`uname -n`"
   testbed="`/sbin/cfgread test_bed`"
   primhost=`/sbin/cfgread node_1_name`
   sechost=`/sbin/cfgread node_2_name`
   if [ "$mynode" = "$primhost" ];
   then
      if [ "$QuietMode" = "false" ];
      then
         echo                 "///////////////////////////////////////////////"
         if [ "$testbed" = "cluster" ];
         then
            echo -n "Do you want to run in on both nodes (y/n) ? "
            read answer
            if [ "$answer" = "y" -o "$answer" = "Y" ];
            then
               DualNode=true
               QuietMode=true
               year=`date +%y`
               month=`date +%m`
               day=`date +%d`
               LogFile=/log/data-${month}-${day}-${year}.tar.gz
               LogFile=
            fi
         fi

         echo -n "Do you want to run in quick mode    (y/n) ? "
         read answer
         if [ "$answer" = "y" -o "$answer" = "Y" ];
         then
            QuickMode=true
         else
            QuickMode=false
         fi
         echo -n "Do you want to limit it to 24 hours (y/n) ? "
         read answer
         if [ "$answer" = "y" -o "$answer" = "Y" ];
         then
            LimitedMode=true
            Ndays=1
         else
            LimitedMode=false
            Ndays=28
         fi
         echo                 "///////////////////////////////////////////////"
      fi

      echo
   fi

   nafoxch="`/sbin/cfgread nafo3`"
   if "$OSV_VERSION" --major compare sys lt V5 || [ "$nafoxch" = "" ] ;
   then
      primcip="`/sbin/cfgread node_1_cip_logical`"
      seccip="`/sbin/cfgread node_2_cip_logical`"
   else
      primcip="`echo $nafoxch   | awk '{ print $4 }'`"
      seccip="`echo $nafoxch    | awk '{ print $5 }'`"
   fi
}

ARGS="$*"
LimitedMode=false

while [ "$#" -gt "0" ];
do
   case $1 in
      -quiet)
         QuietMode=true
         ;;
      -limited)
         LimitedMode=true
         Ndays=1
         ;;
      -quick)
         QuickMode=true
         ;;
      -dual)
         DualNode=true
         ;;
      -remote)
         RemoteMode=true
         ;;
      -master)
         shift
         MasterNode=$1
         ;;
      -log)
         shift
         LogFile=$1
         ;;
       *)
         ;;
   esac
   shift
done
source /unisphere/srx3000/srx/.profile
initvars
checksys
if [ "$testbed" = "cluster" -a "$mynode" != "$primhost" -a "$RemoteMode" = "false" ];
then
   textncolor $BoldCyan "trace8k: O.k you are on secondary node of cluster. Will still proceed."
   ssh $primhost -to ConnectTimeout=8 "$TRACE8k $ARGS -master $mynode" 2>/dev/null
   status=$?
   if [ $status -ne 0 ];
   then
      textncolor $BoldCyan "trace8k: Cannot collect traces from  $primhost. Will run locally."
      textncolor $BoldCyan "trace8k: Run trace8k on $primhost once its available."
      DisaterMode=true
      collect
      status=$?
   fi
else
   collect
   status=$?
fi
exit $status

