Wednesday, July 30, 2014

Linux Memory Troubleshooting

Empty buffer cache
sync && echo 1 > /proc/sys/vm/drop_caches

clean up memory of unnecessary things (Kernerl 2.6.16 or newer)run sync first to flush useful things out to disk!!!
To free pagecache:

echo 1 > /proc/sys/vm/drop_caches    

To free dentries and inodes:

echo 2 > /proc/sys/vm/drop_caches

To free pagecache, dentries and inodes:

echo 3 > /proc/sys/vm/drop_caches
########################### < Memory Troubleshooting > ###########################
TOP 10 MEMORY USERS
ps auxk-rss | head -11 | cut -c1-120
MEMORY % VIA SAR
sar -r | grep -v Average | awk '{print $1" "$2" \t%"$5}'
(unset LANG ;sar -r) |awk '$3~/[0-9]/{total=$3+$2; usedbc=$3-($5+$6); pc_used=(100*usedbc)/total;print $0,pc_used} $3!~/[0-9]/{print $0}' | awk '{print $1" - "$11"%"}'
MEMORY % (RHEL 4)
sar -r | tail -n21 | head -n20 | awk '{ printf "%s -- %3.2f%%\n",$1,($3-$5-$6)*100/($3+$2)}'
DAILY MEMORY AVERAGES
sar -r | grep Average: | awk '{print $1" \t"$4"%"}'
for i in `ls -rt /var/log/sa/ | grep -E "sa[0-9][0-9]"`; do echo -ne "$i -- "; sar -r -f /var/log/sa/$i | grep -Ev "Linux|Average|RESTART|kbmemfree|^$" | awk '{ printf "%3.2f\n",($4-$6-$7)*100/($3+$4)}' | awk '{sum+=$1 } END { printf "Average = %3.2f%%\n",sum/NR}'; done
MONTHLY AVERAGES
for i in `ls /var/log/sa/ | egrep 'sa[0-9][0-9]'` ; do sar -f /var/log/sa/$i -r | grep Average | awk '{print $1" "$4"%"}' ; done
sar -r | grep -v Average | awk '{print $1" "$2" \t"$5"%"}';sar -r | grep Average: | awk '{print $1" \t"$4"%"}'
BETTER FREE REPORT
date ; free -mt ; free -m|awk 'NR==2{print "The total % of "Mem" Used: "($3*100)/$2"%"}'
MEMORY SUMMARY
vmstat -s -S M
echo -ne '\n';echo "===========================================";ps -eo user,%cpu,%mem,rsz,args|sort -rnk4|awk 'BEGIN {printf "%s\t%s\t%s\t%s\t%s\n","USER","%CPU","%MEM","RSZ","COMMAND"}{printf "%s\t%g'%'\t%g'%'\t%d MB\t%-10s\n",$1,$2,$3,$4/1024,$5}'|head -n30;echo "===========================================";echo -e "\n===========================================";vmstat -s -S M|head -n10;echo "===========================================";
CACHE CLEARING
sync; echo "2" > /proc/sys/vm/drop_caches; sleep 10; echo "0" > /proc/sys/vm/drop_caches
APACHE MEMORY USAGE
ps -eo rsz,args | grep httpd | awk ' { SUM += $1 } END { print "Memory used by Apache = "SUM/1024 " Megs" "\nNumber of process runing = " NR "\nAverage of each process mem usage = " SUM/1024/NR " Megs"} '
JAVA MEMORY USAGE
ps afux | grep java | grep -Eo "\-Xmx[0-9]+[m|g] "
</code>
More Memory Investigation
free -mt
vmstat -a -S m
ps auxk-rss |head -11
ps auxk-rss |head -11 | awk '{print $1, $4, $11}'

#Memory %
sar -r | grep -v Average | awk '{print $1" "$2" \t%"$5}'
#Memory Average %
sar -r | grep Average: | awk '{print $1" \t"$4"%"}'

#Monthly Averges
for i in `ls /var/log/sa/ | egrep 'sa[0-9][0-9]'` ; do sar -f /var/log/sa/$i -r | grep Average | awk '{print $1" "$4"%"}' ; done

sar -r | grep -v Average | awk '{print $1" "$2" \t"$5"%"}';sar -r | grep Average: | awk '{print $1" \t"$4"%"}'

date ; free -mt ; free -m|awk 'NR==2{print "The total % of "Mem" Used: "($3*100)/$2"%"}'

#Nimbus Memory %
(unset LANG ;sar -r) |awk '$3~/[0-9]/{total=$3+$2; usedbc=$3-($5+$6); pc_used=(100*usedbc)/total;print $0,pc_used} $3!~/[0-9]/{print $0}' | awk '{print $1" - "$11"%"}'

#Nimbus Memory % (RHEL 4)
sar -r | tail -n21 | head -n20 | awk '{ printf "%s -- %3.2f%%\n",$1,($3-$5-$6)*100/($3+$2)}'

#Summary
vmstat -s -S M

#More detailed overview!
slabtop
#push c to sort by highest cache user!

#Cool trick to clear cache!
[root@269179-db2 ~]# sync
[root@269179-db2 ~]# echo "2" > /proc/sys/vm/drop_caches
[root@269179-db2 ~]# sleep 10
[root@269179-db2 ~]# echo "0" > /proc/sys/vm/drop_caches

#Apache memory:
 ps -eo rsz,args | grep httpd | awk ' { SUM += $1 } END { print "Memory used by Apache = "SUM/1024 " Megs" "\nNumber of process runing = " NR "\nAverage of each process mem usage = " SUM/1024/NR " Megs"} '


low memory:
resize;clear;echo;date;echo "Top 10 Processes by MEM %"; vmstat -a -S m|tail -n1|awk \
'BEGIN {FS=" "}{printf "\nAvail\tActive\tTotal\tPercent Avail\n%sMB\t\
%sMB\t%sMB\t%s\n\n",$4+$5,$6,$4+$5+$6,($4+$5)/($4+$5+$6)*100}';ps -eo \
user,%cpu,%mem,rsz,args|sort -rnk4|awk 'BEGIN {printf "%8s %6s %6s \
%8s     %-10s\n","USER","%CPU","%MEM","RSZ","COMMAND"}{printf "%8s %6s \
%6s %8s MB  %-10s\n",$1,$2,$3,$4/1024,$5}'|head -n10; echo ""; echo "== \
Last Half Hour ==";echo; sar -r|head -n3; sar -r|tail -n4;echo; sar -B|\
head -n3; sar -B|tail -n4;echo;echo "== Current 2 Second Intervals ==";\
echo;sar -r 2 5;echo;sar -B 2 5

#Something with memory and sar
for i in `ls -rt /var/log/sa/ | grep -E "sa[0-9][0-9]"`; do echo -ne "$i -- "; sar -r -f /var/log/sa/$i | grep -Ev "Linux|Average|RESTART|kbmemfree|^$" | awk '{ printf "%3.2f\n",($4-$6-$7)*100/($3+$4)}' | awk '{sum+=$1 } END { printf "Average = %3.2f%%\n",sum/NR}'; done
Show whats using SWAP:
sh swapusage.sh | sort -n -k1 | tac | head -n10
#!/bin/bash
#
# show swap used by processes
#
(for PROCESS in /proc/*/; do
  swapused=$(awk 'BEGIN { total = 0 } /^Swap:[[:blank:]]*[1-9]/ { total = total + $2 } END { print total }' ${PROCESS}/smaps 2>/dev/null || echo 0)
  if [ $swapused -gt 0 ]; then
    /bin/echo -e "${swapused}k\t$(cat ${PROCESS}/cmdline)"
  fi
done ) | sort -nr
#!/bin/bash
# Get current swap usage for all running processes
# Erik Ljungstrom 27/05/2011
SUM=0
OVERALL=0
for DIR in `find /proc/ -maxdepth 1 -type d | egrep "^/proc/[0-9]"` ; do
PID=`echo $DIR | cut -d / -f 3`
PROGNAME=`ps -p $PID -o comm --no-headers`
for SWAP in `grep Swap $DIR/smaps 2>/dev/null| awk '{ print $2 }'`
do
let SUM=$SUM+$SWAP
done
echo "PID=$PID - Swap used: $SUM - ($PROGNAME )"
let OVERALL=$OVERALL+$SUM
SUM=0

done
echo "Overall swap used: $OVERALL"
Pipe thru this for only swap using procs:
| egrep -v "Swap used: 0" |sort -n -k 5
Swap one liner:
SUM=0; OVERALL=0; for DIR in `find /proc/ -maxdepth 1 -type d | egrep "^/proc/[0-9]"` ; do PID=`echo $DIR | cut -d / -f 3

Server monitoring script :)

Script for find out abusive user in server

~~~~~~~~~~~

OUT=$(/usr/local/cpanel/bin/dcpumonview | grep -v Top  | sed -e 's#<[^>]*># #g' | while read i ; do NF=`echo $i | awk {'print NF'}` ; if [[ "$NF" == "5" ]] ; then USER=`echo $i | awk {'print $1'}`; OWNER=`grep -e "^OWNER=" /var/cpanel/users/$USER | cut -d= -f2` ; echo "$OWNER $i"; fi ; done) ; (echo "USER CPU" ; echo "$OUT" | sort -nrk4 | awk '{printf "%s %s%\n",$2,$4}' | head -5) | column -t ;echo;(echo -e "USER MEMORY" ; echo "$OUT" | sort -nrk5 | awk '{printf "%s %s%\n",$2,$5}' | head -5) | column -t ;echo;(echo -e "USER MYSQL" ; echo "$OUT" | sort -nrk6 |
awk '{printf "%s %s%\n",$2,$6}' | head -5) | column -t ;

~~~~~~~~~~~

Finging connections to server 

netstat -pltuna | awk '$6=="LISTEN"{sub(/^.*:+/,"",$4);sub(/^[[:digit:]]+\//,"",$7);idx=sprintf("%s:%05d",$1,$4);ary[idx]=$7;} $6~"^(ESTABLISHED|SYN_RECV|FIN_WAIT2|UNKNOWN)$"{sub(/^.*:(:ffff:)?/,"",$4);sub(/:[[:digit:]]+$/,"",$5);sub(/^::ffff:/,"",$5);idx=sprintf("%s:%05d@%s",$1,$4,$5);cons[idx]++;}END{LIMITS["def"]=30;LIMITS[21]=8;LIMITS[25]=5;LIMITS[26]=5;LIMITS[465]=5;LIMITS[587]=5;CL_NML="\033[0m";CL_WTE="\033[1;37m";CL_GRN="\033[0;32m";CL_YLW="\033[1;36m";CL_RED="\033[1;5;31;22;47m";n=asorti(ary,src);for(i=1;i<=n;i++){split(src[i],meh,/:/);sub(/^0*/,"",meh[2]);print CL_WTE ary[src[i]] CL_NML " " CL_GRN "(" meh[1] ":" meh[2] ")" CL_NML ":";delete nastyhack;for (q in cons){split(q,splt,/@/);if(match(splt[1],src[i])){fmtstr=sprintf("%010d %s",cons[q],splt[2]);nastyhack[fmtstr]=fmtstr;}}r=asort(nastyhack);zerocount=match(nastyhack[r],/[^0]/);for (m=1;m<=r;m++){nastyhack[m]=substr(nastyhack[m],zerocount);split(nastyhack[m],brg,/ /);printf CL_YLW brg[1] CL_NML " ";port=meh[2];if(!(port in LIMITS)) port="def";if (brg[1]>LIMITS[port]) printf CL_RED;print brg[2] CL_NML;}}}'



Blocking Ips DDOS

 ~~~~~~~~~~~~

grep "Port Flood" /var/log/messages | grep "Jul 17" | awk '{ print $12 }' | cut -d = -f2 | sort | uniq -c | sort -n > /root/testflood

    cat /root/testflood

    while read line; do number=$(echo $line | awk {'print $1'}); ip=$(echo $line | awk {'print $2'}); if [ $number -gt 500 ]; then csf -d $ip "Wp attack"; fi; done < /root/testflood

~~~~~~~~~~~~