free,sar,iostat,vmstat,pidstat,ipcs,watch,ss
http://blog.chinaunix.net/uid-20787846-id-1842125.html  系统性能监控工具
内存监测每2s以M,K,B为单位分别显示



[root@test1 ~]# free -o
             total       used       free     shared    buffers     cached
Mem:       3943580    3890992      52588          0     184736    2178076
Swap:      6291448          0    6291448
[root@test1 ~]# free -ot
             total       used       free     shared    buffers     cached
Mem:       3943580    3891984      51596          0     184764    2178548
Swap:      6291448          0    6291448
Total:    10235028    3891984    6343044


[root@221-comecs mnt]# free -s2 -m total used free shared buffers cached Mem: 995 562 433 0 95 337 -/+ buffers/cache: 129 866 Swap: 1635 0 1635 total used free shared buffers cached Mem: 995 562 433 0 95 337 -/+ buffers/cache: 129 866 Swap: 1635 0 1635 [root@221-comecs mnt]# free -s2 total used free shared buffers cached Mem: 1019820 575496 444324 0 97448 345320 -/+ buffers/cache: 132728 887092 Swap: 1675256 0 1675256 total used free shared buffers cached Mem: 1019820 575504 444316 0 97448 345320 -/+ buffers/cache: 132736 887084 Swap: 1675256 0 1675256 [root@221-comecs mnt]# free -s2 -b total used free shared buffers cached Mem: 1044295680 589180928 455114752 0 99786752 353607680 -/+ buffers/cache: 135786496 908509184 Swap: 1715462144 0 1715462144 total used free shared buffers cached Mem: 1044295680 589189120 455106560 0 99786752 353607680 -/+ buffers/cache: 135794688 908500992 Swap: 1715462144 0 1715462144
[root@109-com1 ~]# yum install iotop
[root@109-com1 asterisk]# watch -n1 -d free
Every 1.0s: free                                                                                                                     Thu Oct  9 14:53:01 2014

             total       used       free     shared    buffers     cached
Mem:       1019820     933400      86420          0     112452     515660
-/+ buffers/cache:     305288     714532
Swap:      1675256        456    1674800
[root@109-com1 asterisk]# watch -d vmstat
Every 2.0s: vmstat                                                                                                                   Thu Oct  9 14:58:28 2014

procs -----------memory---------- ---swap-- -----io---- --system-- -----cpu-----
 r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa st
 0  0    456  74072 112636 527608    0    0     0    17   21   40  0  0 99  0  0

[root@250-shiyan bin]# ps aux|head -1;ps aux|sort -rnk6|head  以RSS即第六列为准由大到小排序
[root@oracle5A flt]# mpstat -P ALL 1
Linux 2.6.32-431.el6.x86_64 (oracle5A)  10/15/2014      _x86_64_        (4 CPU)

06:27:23 PM  CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest   %idle
06:27:24 PM  all    0.75    0.00    0.25    0.00    0.00    0.00    0.00    0.00   99.00
06:27:24 PM    0    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00  100.00
06:27:24 PM    1    0.99    0.00    0.99    0.00    0.00    0.00    0.00    0.00   98.02
06:27:24 PM    2    1.00    0.00    1.00    0.00    0.00    0.00    0.00    0.00   98.00
06:27:24 PM    3    1.00    0.00    1.00    0.00    0.00    0.00    0.00    0.00   98.00


[root@221-comecs ~]# vmstat -a 1
procs -----------memory---------- ---swap-- -----io---- --system-- -----cpu-----
 r  b   swpd   free  inact active   si   so    bi    bo   in   cs us sy id wa st
 0  0  11244  72396 570260 272184    0    0     2     8    0    0  1  1 98  0  0
 0  0  11244  72264 570284 272252    0    0     0   472 3906 8222  2  1 97  0  0
[root@221-comecs ~]# vmstat -f
     13839576 forks
[root@221-comecs ~]# vmstat -ta 1
procs -----------memory---------- ---swap-- -----io---- --system-- -----cpu------ ---timestamp---
 r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa st
 0  0  11244  69196  95452 548540    0    0     2     8    0    0  1  1 98  0  0        2015-11-16 14:02:31 CST
 0  0  11244  69188  95452 548560    0    0     0     0 3927 8126  2  1 97  0  0        2015-11-16 14:02:32 CST
[root@localhost ~]# vmstat -s
       502168  total memory
        91576  used memory
        23196  active memory
        25396  inactive memory
[root@localhost ~]# vmstat -m
Cache                       Num  Total   Size  Pages
nf_conntrack_expect           0      0    240     16
nf_conntrack_ffffffff81b18540      0      0    312     12
fib6_nodes                   22     59     64     59
ip6_dst_cache                13     20    384     10
ndisc_cache                   1     15    256     15

ip6_mrt_cache                 0      0    128     30
RAWv6                         4      4   1024      4
UDPLITEv6                     0      0   1024      4
[root@localhost ~]# vmstat -p /dev/sda2 1
sda2          reads   read sectors  writes    requested writes
                1794      77490        154       3232
                1794      77490        154       3232
                1794      77490        154       3232
加w会变宽,不会看起来那么拥挤
[root@221-comecs ~]# vmstat 1 -w
procs -------------------memory------------------ ---swap-- -----io---- --system-- -----cpu-------
 r  b       swpd       free       buff      cache   si   so    bi    bo   in   cs  us sy  id wa st
 2  0      11244      70540      94760     545484    0    0     2     8    0    0   1  1  98  0  0
 0  0      11244      71332      94760     545492    0    0     0     0 6286 13875   2  2  96  0  0
 0  0      11244      71340      94760     545516    0    0     0     0 9728 22204   2  2  96  0  0


[root@221-comecs ~]# iostat -k -c 1
Linux 2.6.32-431.el6.x86_64 (221-comecs)        12/11/2014      _x86_64_        (4 CPU)

avg-cpu:  %user   %nice %system %iowait  %steal   %idle
           0.26    0.00    0.31    0.07    0.00   99.35

avg-cpu:  %user   %nice %system %iowait  %steal   %idle
           0.77    0.00    0.51    0.00    0.00   98.72
[root@221-comecs ~]# iostat -k -d 1
Linux 2.6.32-431.el6.x86_64 (221-comecs)        12/11/2014      _x86_64_        (4 CPU)

Device:            tps    kB_read/s    kB_wrtn/s    kB_read    kB_wrtn
sda               1.23         3.36       103.60    6461574  199095821
dm-0             25.99         3.36       103.60    6457057  199095812
dm-1              0.00         0.00         0.00       1188          0

Device:            tps    kB_read/s    kB_wrtn/s    kB_read    kB_wrtn
sda              21.78         0.00      8273.27          0       8356
dm-0           2068.32         0.00      8273.27          0       8356
dm-1              0.00         0.00         0.00          0          0
[root@221-comecs ~]# iostat -k -N 1
Linux 2.6.32-431.el6.x86_64 (221-comecs)        12/11/2014      _x86_64_        (4 CPU)

avg-cpu:  %user   %nice %system %iowait  %steal   %idle
           0.26    0.00    0.31    0.07    0.00   99.35

Device:            tps    kB_read/s    kB_wrtn/s    kB_read    kB_wrtn
sda               1.23         3.36       103.61    6461602  199139673
VolGroup-lv_root    25.99         3.36       103.61    6457085  199139664
VolGroup-lv_swap     0.00         0.00         0.00       1188          0
[root@221-comecs ~]# iostat  -cx 1
Linux 2.6.32-431.el6.x86_64 (221-comecs)        12/11/2014      _x86_64_        (4 CPU)

avg-cpu:  %user   %nice %system %iowait  %steal   %idle
           0.26    0.00    0.31    0.07    0.00   99.35

Device:         rrqm/s   wrqm/s     r/s     w/s   rsec/s   wsec/s avgrq-sz avgqu-sz   await  svctm  %util
sda               0.00    24.76    0.09    1.14     6.72   207.21   174.21     0.02   19.55   4.53   0.56
dm-0              0.00     0.00    0.09   25.90     6.72   207.21     8.23     1.30   50.10   0.21   0.56
dm-1              0.00     0.00    0.00    0.00     0.00     0.00     8.00     0.00    1.10   0.79   0.00
[root@221-comecs ~]# iostat -k -p dm-0 1 -t
Linux 2.6.32-431.el6.x86_64 (221-comecs)        12/11/2014      _x86_64_        (4 CPU)

12/11/2014 05:10:02 PM
avg-cpu:  %user   %nice %system %iowait  %steal   %idle
           0.26    0.00    0.31    0.07    0.00   99.35

Device:            tps    kB_read/s    kB_wrtn/s    kB_read    kB_wrtn
dm-0             26.00         3.36       103.64    6457133  199205400
[root@221-comecs ~]# iostat -k -n 1 -t
Linux 2.6.32-431.el6.x86_64 (221-comecs)        12/11/2014      _x86_64_        (4 CPU)

12/11/2014 05:11:40 PM
Filesystem:               rkB_nor/s    wkB_nor/s    rkB_dir/s    wkB_dir/s    rkB_svr/s    wkB_svr/s     ops/s    rops/s    wops/s
192.168.2.2:/fileserv          0.00         0.00         0.00         0.00         0.00         0.00      0.00      0.00      0.00
[root@221-comecs ~]# sar --help
Usage: sar [ options ] [ <interval> [ <count> ] ]
Options are:
[ -A ] [ -b ] [ -B ] [ -C ] [ -d ] [ -h ] [ -m ] [ -p ] [ -q ] [ -r ] [ -R ]
[ -S ] [ -t ] [ -u [ ALL ] ] [ -v ] [ -V ] [ -w ] [ -W ] [ -y ]
[ -I { <int> [,...] | SUM | ALL | XALL } ] [ -P { <cpu> [,...] | ALL } ]
[ -j { ID | LABEL | PATH | UUID | ... } ] [ -n { <keyword> [,...] | ALL } ]
[ -o [ <filename> ] | -f [ <filename> ] ] [ --legacy ]
[ -i <interval> ] [ -s [ <hh:mm:ss> ] ] [ -e [ <hh:mm:ss> ] ]
[root@221-comecs ~]# sar -h
Usage: sar [ options ] [ <interval> [ <count> ] ]
Main options and reports:
        -b      I/O and transfer rate statistics
        -B      Paging statistics
        -d      Block device statistics
        -I { <int> | SUM | ALL | XALL }
                Interrupts statistics
        -m      Power management statistics
        -n { <keyword> [,...] | ALL }
                Network statistics
                Keywords are:
                DEV     Network interfaces
                EDEV    Network interfaces (errors)
                NFS     NFS client
                NFSD    NFS server
                SOCK    Sockets (v4)
                IP      IP traffic      (v4)
                EIP     IP traffic      (v4) (errors)
                ICMP    ICMP traffic    (v4)
                EICMP   ICMP traffic    (v4) (errors)
                TCP     TCP traffic     (v4)
                ETCP    TCP traffic     (v4) (errors)
                UDP     UDP traffic     (v4)
                SOCK6   Sockets (v6)
                IP6     IP traffic      (v6)
                EIP6    IP traffic      (v6) (errors)
                ICMP6   ICMP traffic    (v6)
                EICMP6  ICMP traffic    (v6) (errors)
                UDP6    UDP traffic     (v6)
        -q      Queue length and load average statistics
        -r      Memory utilization statistics
        -R      Memory statistics
        -S      Swap space utilization statistics
        -u [ ALL ]
                CPU utilization statistics
        -v      Kernel table statistics
        -w      Task creation and system switching statistics
        -W      Swapping statistics
        -y      TTY device statistics

不带选项执行会以10分钟为间隔报告自午夜起当天的CPU利用率。
[root@221-comecs ~]# sar
Linux 2.6.32-431.el6.x86_64 (221-comecs)        12/11/2014      _x86_64_        (4 CPU)

12:00:01 AM     CPU     %user     %nice   %system   %iowait    %steal     %idle
12:10:01 AM     all      0.12      0.00      0.14      0.07      0.00     99.66
12:20:01 AM     all      0.10      0.00      0.15      0.08      0.00     99.68
......
04:50:01 PM     all      0.99      0.00      0.64      0.05      0.00     98.32
05:00:01 PM     all      1.10      0.00      0.70      0.08      0.00     98.12
05:10:01 PM     all      0.78      0.00      0.80      0.52      0.00     97.91
05:20:01 PM     all      0.74      0.00      0.59      0.07      0.00     98.61
Average:        all      0.44      0.00      0.40      0.09      0.00     99.07
[root@ipt ~]# sar -m 1
Linux 2.6.32-431.el6.x86_64 (ipt)       10/17/2014      _x86_64_        (2 CPU)

09:46:58 PM     CPU       MHz
09:46:59 PM     all   2493.76
09:47:00 PM     all   2493.76
[root@ipt ~]# sar -C 1
Linux 2.6.32-431.el6.x86_64 (ipt)       10/17/2014      _x86_64_        (2 CPU)

09:46:33 PM     CPU     %user     %nice   %system   %iowait    %steal     %idle
09:46:34 PM     all      0.00      0.00      0.50      0.00      0.00     99.50
09:46:35 PM     all      0.00      0.00      0.00      0.00      0.00    100.00
[root@ipt ~]# sar -B 1
Linux 2.6.32-431.el6.x86_64 (ipt)       10/17/2014      _x86_64_        (2 CPU)

09:46:12 PM  pgpgin/s pgpgout/s   fault/s  majflt/s  pgfree/s pgscank/s pgscand/s pgsteal/s    %vmeff
09:46:13 PM      0.00      8.08     37.37      0.00    111.11      0.00      0.00      0.00      0.00
09:46:14 PM      0.00      0.00     40.00      0.00    110.00      0.00      0.00      0.00      0.00
pgpgin/s 每秒从磁盘分到内存中的页面数,paged in    置入分页;页入    the system paged in from disk
pgpgout/s 每秒从内存分到磁盘的页面数,paged out    页出;移出分页    the system paged out to disk
fault/s 每秒的页面故障数
majflt/s 每秒的主要页面故障数
With older kernels, a block is of indeterminate size.
indeterminate    adj. 不确定的;模糊的;含混的
[root@ipt
~]# sar -b 1 Linux 2.6.32-431.el6.x86_64 (ipt) 10/17/2014 _x86_64_ (2 CPU) 09:45:42 PM tps rtps wtps bread/s bwrtn/s 09:45:43 PM 0.00 0.00 0.00 0.00 0.00 09:45:44 PM 0.00 0.00 0.00 0.00 0.00 查看系统内存使用情况 [root@221-comecs ~]# sar -r 1 Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU) 05:25:18 PM kbmemfree kbmemused %memused kbbuffers kbcached kbcommit %commit 05:25:19 PM 68656 951164 93.27 68072 682032 194064 7.20 05:25:20 PM 68284 951536 93.30 68076 682228 194064 7.20 05:25:21 PM 68036 951784 93.33 68076 682568 194064 7.20 [root@221-comecs ~]# sar -S 1 Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU) 05:36:40 PM kbswpfree kbswpused %swpused kbswpcad %swpcad 05:36:41 PM 1675256 0 0.00 0 0.00 05:36:42 PM 1675256 0 0.00 0 0.00 05:36:43 PM 1675256 0 0.00 0 0.00 查看系统磁盘的读写情况。 [root@221-comecs ~]# sar -d 1 Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU) 05:26:26 PM DEV tps rd_sec/s wr_sec/s avgrq-sz avgqu-sz await svctm %util 05:26:27 PM dev8-0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 05:26:27 PM dev253-0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 05:26:27 PM dev253-1 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00

[root@221-comecs ~]# sar -dp 1
Linux 2.6.32-431.el6.x86_64 (221-comecs)        11/16/2015      _x86_64_        (4 CPU)

11:07:54 AM       DEV       tps  rd_sec/s  wr_sec/s  avgrq-sz  avgqu-sz     await     svctm     %util
11:07:55 AM       sda      4.21      0.00     42.11     10.00      0.05     11.00      3.25      1.37
11:07:55 AM VolGroup-lv_root      5.26      0.00     42.11      8.00      0.05     10.40      2.60      1.37
11:07:55 AM VolGroup-lv_swap      0.00      0.00      0.00      0.00      0.00      0.00      0.00      0.00
查看2号cpu及所有cpu使用情况 [root@221-comecs ~]# sar -P 2 1 Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU) 05:27:50 PM CPU %user %nice %system %iowait %steal %idle 05:27:51 PM 2 0.00 0.00 0.99 0.00 0.00 99.01 05:27:52 PM 2 2.02 0.00 0.00 0.00 0.00 97.98 05:27:53 PM 2 0.00 0.00 0.00 0.00 0.00 100.00 ^C [root@221-comecs ~]# sar -P ALL 1 Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU) 05:27:59 PM CPU %user %nice %system %iowait %steal %idle 05:28:00 PM all 0.76 0.00 0.76 0.25 0.00 98.24 05:28:00 PM 0 3.03 0.00 1.01 0.00 0.00 95.96 05:28:00 PM 1 1.01 0.00 1.01 1.01 0.00 96.97 05:28:00 PM 2 0.00 0.00 0.00 0.00 0.00 100.00 05:28:00 PM 3 0.00 0.00 0.00 0.00 0.00 100.00 [root@221-comecs ~]# sar -u 1 Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU) 05:29:27 PM CPU %user %nice %system %iowait %steal %idle 05:29:28 PM all 0.50 0.00 0.50 0.00 0.00 99.00 05:29:29 PM all 0.75 0.00 0.50 0.00 0.00 98.75
[root@rac02 ~]# sar -n DEV 1 0|egrep "IFACE|lo"
02:57:18 PM     IFACE   rxpck/s   txpck/s   rxbyt/s   txbyt/s   rxcmp/s   txcmp/s  rxmcst/s
02:57:19 PM        lo   2223.00   2223.00 282223.00 282223.00      0.00      0.00      0.00
02:57:19 PM     IFACE   rxpck/s   txpck/s   rxbyt/s   txbyt/s   rxcmp/s   txcmp/s  rxmcst/s
02:57:20 PM        lo     59.41     59.41  13728.71  13728.71      0.00      0.00      0.00
[root@221-comecs ~]# sar -n DEV 1 Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU) 05:30:50 PM IFACE rxpck/s txpck/s rxkB/s txkB/s rxcmp/s txcmp/s rxmcst/s 05:30:51 PM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 05:30:51 PM eth0 23.00 24.00 3.67 2.34 0.00 0.00 0.00 [root@221-comecs ~]# sar -n EDEV 1 Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU) 05:32:24 PM IFACE rxerr/s txerr/s coll/s rxdrop/s txdrop/s txcarr/s rxfram/s rxfifo/s txfifo/s 05:32:25 PM lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 05:32:25 PM eth0 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 [root@221-comecs ~]# sar -n SOCK 1 Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU) 05:31:47 PM totsck tcpsck udpsck rawsck ip-frag tcp-tw 05:31:48 PM 147 15 11 0 0 0 05:31:49 PM 147 15 11 0 0 0 05:31:50 PM 147 15 11 0 0 0 [root@221-comecs ~]# sar -v 1 Linux 2.6.32-431.el6.x86_64 (221-comecs) 12/11/2014 _x86_64_ (4 CPU) 05:35:13 PM dentunusd file-nr inode-nr pty-nr 05:35:14 PM 1806 544 6747 34 05:35:15 PM 1806 544 6747 34 05:35:16 PM 1806 544 6747 34

###收集数据
[root@250-shiyan sa]# /usr/lib64/sa/sadc 1 10 sadc00
###sadc 是把数据写在一个二进制的文件中,如果想查看数据内容,需要用sadf工具来显示;如果不指定文件名,默认是屏幕。
###查看数据    两条命令都可以查看sadc00文件的数据,但sadf不太直观
[root@250-shiyan sa]# sar -f sadc00
Linux 2.6.32-431.el6.x86_64 (250-shiyan)        12/12/2014      _x86_64_        (1 CPU)

11:28:13 AM     CPU     %user     %nice   %system   %iowait    %steal     %idle
11:28:14 AM     all      0.00      0.00      0.00      1.00      0.00     99.00
11:28:15 AM     all      0.00      0.00      0.00      0.00      0.00    100.00
11:28:16 AM     all      0.99      0.00      0.00      0.00      0.00     99.01
11:28:17 AM     all      0.00      0.00      0.00      0.00      0.00    100.00
11:28:18 AM     all      0.00      0.00      0.99      0.00      0.00     99.01
11:28:19 AM     all      0.00      0.00      0.00      0.00      0.00    100.00
11:28:20 AM     all      0.00      0.00      0.00      0.00      0.00    100.00
11:28:21 AM     all      0.00      0.00      0.00      0.00      0.00    100.00
11:28:22 AM     all      0.00      0.00      0.00      1.00      0.00     99.00
Average:        all      0.11      0.00      0.11      0.22      0.00     99.56
[root@250-shiyan sa]# sadf sadc00
250-shiyan      1       1418354966      all     %user   0.99
250-shiyan      1       1418354966      all     %nice   0.00
250-shiyan      1       1418354966      all     %system 0.00
250-shiyan      1       1418354966      all     %iowait 0.99
250-shiyan      1       1418354966      all     %steal  0.00
250-shiyan      1       1418354966      all     %idle   98.0
[root@109-com1 ~]# pidstat 1
Linux 2.6.32-431.el6.x86_64 (109-com1)  12/12/2014      _x86_64_        (4 CPU)

05:02:25 PM       PID    %usr %system  %guest    %CPU   CPU  Command
05:02:26 PM      5545    8.91    8.91    0.00   17.82     0  asterisk
05:02:26 PM     10216    0.00    1.98    0.00    1.98     1  FLTService
05:02:26 PM     12611    0.00    0.99    0.00    0.99     1  pidstat

[root@109-com1 ~]# pidstat -I 1
Linux 2.6.32-431.el6.x86_64 (109-com1)  12/12/2014      _x86_64_        (4 CPU)

05:01:25 PM       PID    %usr %system  %guest    %CPU   CPU  Command
05:01:26 PM      5545    9.90    7.92    0.00    4.59     0  asterisk
05:01:26 PM     10216    0.99    1.98    0.00    0.77     1  FLTService
05:01:26 PM     12595    0.00    0.99    0.00    0.26     1  pidstat
[root@109-com1 ~]# pidstat -h 1
Linux 2.6.32-431.el6.x86_64 (109-com1)  12/12/2014      _x86_64_        (4 CPU)

#      Time       PID    %usr %system  %guest    %CPU   CPU  Command
 1418374844        20    0.00    0.99    0.00    0.99     1  events/1
 1418374844      5545    9.90    9.90    0.00   19.80     0  asterisk
 1418374844     10216    0.99    1.98    0.00    2.97     1  FLTService
 1418374844     12568    0.00    1.98    0.00    1.98     2  pidstat
[root@109-com1 ~]# pidstat 1 -l
Linux 2.6.32-431.el6.x86_64 (109-com1)  12/12/2014      _x86_64_        (4 CPU)

05:03:08 PM       PID    %usr %system  %guest    %CPU   CPU  Command
05:03:09 PM      5545    8.91    9.90    0.00   18.81     0  /usr/sbin/asterisk -f -vvvg -c
05:03:09 PM     10216    0.99    2.97    0.00    3.96     1  ./FLTService
05:03:09 PM     12628    0.00    0.99    0.00    0.99     1  pidstat 1 -l
[root@109-com1 ~]# pidstat 1 -C FL
Linux 2.6.32-431.el6.x86_64 (109-com1)  12/12/2014      _x86_64_        (4 CPU)

04:58:20 PM       PID    %usr %system  %guest    %CPU   CPU  Command
04:58:21 PM     10216    0.00    2.97    0.00    2.97     1  FLTService

04:58:21 PM       PID    %usr %system  %guest    %CPU   CPU  Command
04:58:22 PM     10216    1.00    2.00    0.00    3.00     1  FLTService
[root@109-com1 ~]# pidstat -d 1
Linux 2.6.32-431.el6.x86_64 (109-com1)  12/12/2014      _x86_64_        (4 CPU)

05:00:06 PM       PID   kB_rd/s   kB_wr/s kB_ccwr/s  Command
05:00:07 PM       347      0.00     87.13      0.00  jbd2/dm-0-8
05:00:07 PM      5545      0.00     39.60      0.00  asterisk
05:00:07 PM     10216      0.00     11.88      0.00  FLTService

05:00:07 PM       PID   kB_rd/s   kB_wr/s kB_ccwr/s  Command
05:00:08 PM      5545      0.00     32.00      0.00  asterisk
[root@109-com1 ~]# pidstat 1 -r
Linux 2.6.32-431.el6.x86_64 (109-com1)  12/12/2014      _x86_64_        (4 CPU)

05:04:27 PM       PID  minflt/s  majflt/s     VSZ    RSS   %MEM  Command
05:04:28 PM      5545      4.95      0.00 2387204 193976  19.02  asterisk
05:04:28 PM     12666    253.47      0.00  101096    932   0.09  pidstat

05:04:30 PM       PID  minflt/s  majflt/s     VSZ    RSS   %MEM  Command
05:04:31 PM      4790      1.00      0.00  197676   2220   0.22  snmpd
05:04:31 PM      5545      8.00      0.00 2387204 193976  19.02  asterisk
05:04:31 PM     12666    254.00      0.00  101096    964   0.09  pidstat
[root@109-com1 ~]# pidstat 1 -p 10216
Linux 2.6.32-431.el6.x86_64 (109-com1)  12/12/2014      _x86_64_        (4 CPU)

05:05:30 PM       PID    %usr %system  %guest    %CPU   CPU  Command
05:05:31 PM     10216    0.00    3.00    0.00    3.00     0  FLTService
05:05:32 PM     10216    1.00    3.00    0.00    4.00     0  FLTService
05:05:33 PM     10216    1.00    3.00    0.00    4.00     0  FLTService
05:05:34 PM     10216    1.00    3.00    0.00    4.00     0  FLTService
[root@109-com1 ~]# pidstat 1 -p 5545 -T ALL
Linux 2.6.32-431.el6.x86_64 (109-com1)  12/12/2014      _x86_64_        (4 CPU)

05:22:26 PM       PID    %usr %system  %guest    %CPU   CPU  Command
05:22:27 PM      5545   10.00   10.00    0.00   20.00     1  asterisk

05:22:26 PM       PID    usr-ms system-ms  guest-ms  Command
05:22:27 PM      5545       100       100         0  asterisk
05:22:28 PM      5545    8.00   10.00    0.00   18.00     1  asterisk
05:22:28 PM      5545        80       100         0  asterisk
[root@109-com1 ~]# pidstat 1 -r  -p 5545 -T ALL
Linux 2.6.32-431.el6.x86_64 (109-com1)  12/12/2014      _x86_64_        (4 CPU)

05:23:21 PM       PID  minflt/s  majflt/s     VSZ    RSS   %MEM  Command
05:23:22 PM      5545      5.00      0.00 2387204 193980  19.02  asterisk

05:23:21 PM       PID minflt-nr majflt-nr  Command
05:23:22 PM      5545         5         0  asterisk
[root@aster3 asterisk]# ipcs -l

------ Shared Memory Limits --------
max number of segments = 4096
max seg size (kbytes) = 67108864
max total shared memory (kbytes) = 17179869184
min seg size (bytes) = 1

------ Semaphore Limits --------
max number of arrays = 128
max semaphores per array = 250
max semaphores system wide = 32000
max ops per semop call = 32
semaphore max value = 32767

------ Messages: Limits --------
max queues system wide = 973
max size of message (bytes) = 65536
default max size of queue (bytes) = 65536

[root@aster3 asterisk]# ipcs -a ------ Shared Memory Segments -------- key shmid owner perms bytes nattch status 0x74000417 983040 root 600 4 0 0x740003ec 1638401 root 600 4 0 0x00000000 1998850 root 644 52 2 0x740003eb 1605635 root 600 4 0 0x00000000 2031620 root 644 16384 2 0x00000000 2064389 root 644 268 2 ------ Semaphore Arrays -------- key semid owner perms nsems 0x000000a7 0 root 600 1 0x00000000 32769 apache 600 1 0x00000000 262146 apache 600 1 0x00000000 294915 apache 600 1 0x00000000 327684 apache 600 1 0x00000000 360453 apache 600 1 0x00000000 393222 apache 600 1 0x00000000 425991 apache 600 1 ------ Message Queues -------- key msqid owner perms used-bytes messages [root@aster3 asterisk]# ipcs -s ------ Semaphore Arrays -------- key semid owner perms nsems 0x000000a7 0 root 600 1 0x00000000 32769 apache 600 1 0x00000000 262146 apache 600 1 0x00000000 294915 apache 600 1 0x00000000 327684 apache 600 1 0x00000000 360453 apache 600 1 0x00000000 393222 apache 600 1 0x00000000 425991 apache 600 1 [root@aster3 asterisk]# ipcs -m ------ Shared Memory Segments -------- key shmid owner perms bytes nattch status 0x74000417 983040 root 600 4 0 0x740003ec 1638401 root 600 4 0 0x00000000 1998850 root 644 52 2 0x740003eb 1605635 root 600 4 0 0x00000000 2031620 root 644 16384 2 0x00000000 2064389 root 644 268 2 [root@aster3 asterisk]# ipcs -mi 2064389 Shared memory Segment shmid=2064389 uid=0 gid=0 cuid=0 cgid=0 mode=0644 access_perms=0644 bytes=268 lpid=3357 cpid=3340 nattch=2 att_time=Mon Dec 8 10:35:54 2014 det_time=Mon Dec 8 10:35:54 2014 change_time=Mon Dec 8 10:35:54 2014 [root@aster3 asterisk]# ipcs -mu ------ Shared Memory Status -------- segments allocated 6 pages allocated 9 pages resident 9 pages swapped 0 Swap performance: 0 attempts 0 successes [root@aster3 asterisk]# ipcs -ml ------ Shared Memory Limits -------- max number of segments = 4096 max seg size (kbytes) = 4194303 max total shared memory (kbytes) = 1073741824 min seg size (bytes) = 1
[root@221-comecs ~]# watch -n1 -d "ss -no"
Every 1.0s: ss -no                                                                                                                    Thu Dec 18 11:18:00 2014

State      Recv-Q Send-Q        Local Address:Port          Peer Address:Port
ESTAB      0      0             192.168.2.221:10001         124.89.70.40:4776   timer:(keepalive,103min,0)
ESTAB      0      1506          192.168.2.221:10001        36.46.211.248:1872   timer:(on,235ms,0)
ESTAB      0      1506          192.168.2.221:10001        36.46.211.248:4482   timer:(on,165ms,0)
ESTAB      0      0             192.168.2.221:22            192.168.2.84:54625  timer:(keepalive,114min,0)
ESTAB      0      149           192.168.2.221:10001        36.46.211.248:1432   timer:(on,181ms,0)
ESTAB      0      149           192.168.2.221:10001        36.46.211.248:20403  timer:(on,344ms,0)
ESTAB      0      0             192.168.2.221:10001        36.46.211.248:4277   timer:(keepalive,78min,0)
ESTAB      0      1506          192.168.2.221:10001        36.46.211.248:4932   timer:(on,321ms,0)
ESTAB      0      0             192.168.2.221:53198          192.168.2.2:1521
ESTAB      0      0             192.168.2.221:10001       123.138.91.202:2702   timer:(keepalive,34min,0)
ESTAB      0      149           192.168.2.221:10001        117.36.195.67:12850  timer:(on,343ms,0)
ESTAB      0      306           192.168.2.221:10001        36.46.211.248:2644   timer:(on,231ms,0)
ESTAB      0      0             192.168.2.221:10001       113.138.207.39:2302   timer:(keepalive,67min,0)
ESTAB      0      0             192.168.2.221:10001       210.74.130.138:50607  timer:(keepalive,109min,0)
ESTAB      0      0             192.168.2.221:53201          192.168.2.2:1521
[root@109-com1 ~]# watch -n1 -d "ss -n -o state fin-wait-1"
Every 1.0s: ss -n -o state fin-wait-1                                                                                                 Thu Dec 18 11:23:17 2014

Recv-Q Send-Q             Local Address:Port               Peer Address:Port
0      1                  192.168.2.109:10000            223.104.11.173:30328  timer:(on,4.409ms,3)
0      1                  192.168.2.109:10000             117.136.25.78:50720  timer:(on,3.505ms,0)
0      1                  192.168.2.109:10000            223.104.11.172:26123  timer:(on,34sec,6)
0      1                  192.168.2.109:10000            117.136.25.185:6084   timer:(on,20sec,5)
0      1                  192.168.2.109:10000            117.136.25.189:60125  timer:(on,4.732ms,2)
0      21                 192.168.2.109:10000             117.136.64.85:63212  timer:(on,39sec,8)
0      1                  192.168.2.109:10000            223.104.11.176:27962  timer:(on,45sec,5)
0      1                  192.168.2.109:10000            223.104.11.180:14531  timer:(on,1min1sec,6)
0      1                  192.168.2.109:10000              117.136.4.27:14568  timer:(on,1min5sec,6)
0      1                  192.168.2.109:10000              117.136.4.16:26374  timer:(on,19sec,2)
0      1                  192.168.2.109:10000              117.136.4.22:24919  timer:(on,4.621ms,6)
0      1                  192.168.2.109:10000            117.136.38.175:38252  timer:(on,1min1sec,5)
0      1                  192.168.2.109:10000            117.136.25.180:52928  timer:(on,5.700ms,6)
0      1                  192.168.2.109:10000            117.136.25.220:43257  timer:(on,5.912ms,6)
[root@109-com1 ~]# watch -n1 -d "netstat -not"
Every 1.0s: netstat -not                                                                                                              Thu Dec 18 11:48:52 2014

Active Internet connections (w/o servers)
Proto Recv-Q Send-Q Local Address               Foreign Address             State       Timer
tcp        0      0 192.168.2.109:10000         117.136.3.10:4607           SYN_RECV    on (16.97/5/0)
tcp        0      0 192.168.2.109:10000         223.104.11.85:16549         ESTABLISHED keepalive (1126.16/0/0)
tcp        0      0 192.168.2.109:10000         117.136.25.189:15734        ESTABLISHED keepalive (3639.54/0/0)
tcp        0      0 192.168.2.109:10000         117.136.25.76:53721         ESTABLISHED keepalive (990.24/0/0)
tcp        0      0 192.168.2.109:10000         117.136.25.179:14765        ESTABLISHED keepalive (2948.78/0/0)
tcp        0      0 192.168.2.109:10000         117.136.25.136:30413        ESTABLISHED keepalive (5343.41/0/0)
tcp        0      0 192.168.2.109:10000         223.104.11.182:35807        ESTABLISHED keepalive (1776.77/0/0)
tcp        0      0 192.168.2.109:10000         117.136.25.189:42258        ESTABLISHED keepalive (2645.11/0/0)
tcp        0      0 192.168.2.109:10000         223.104.11.172:26158        ESTABLISHED keepalive (7132.59/0/0)
tcp        0      0 192.168.2.109:10000         223.104.11.85:6855          ESTABLISHED keepalive (6403.83/0/0)
tcp        0      0 192.168.2.109:10000         223.104.11.84:41843         ESTABLISHED keepalive (3607.52/0/0)
tcp        0      0 192.168.2.109:10000         223.104.11.175:46953        ESTABLISHED keepalive (3189.19/0/0)
tcp        0      0 192.168.2.109:10000         117.136.50.35:57643         ESTABLISHED keepalive (6432.29/0/0)
tcp        0     20 192.168.2.109:10000         117.136.25.142:45223        ESTABLISHED on (5.28/1/0)
tcp        0     20 192.168.2.109:10000         117.136.25.184:49946        ESTABLISHED on (29.47/5/0)
tcp        0      0 192.168.2.109:10000         223.104.11.176:64584        ESTABLISHED keepalive (1179.13/0/0)
[root@222-comecs ~]# watch -n1 -d "ss -ni"
Every 1.0s: ss -ni                                                                                                                    Mon Dec 22 11:57:39 2014

State      Recv-Q Send-Q        Local Address:Port          Peer Address:Port
ESTAB      0      0             192.168.2.222:46839        192.168.2.220:1521
         cubic wscale:7,6 rto:208 rtt:8.25/11 ato:40 cwnd:10 send 14.0Mbps rcv_rtt:6 rcv_space:14600
ESTAB      0      0             192.168.2.222:46797        192.168.2.220:1521
         cubic wscale:7,6 rto:201 rtt:1.875/0.75 ato:40 cwnd:10 send 61.8Mbps rcv_rtt:8 rcv_space:14600
ESTAB      0      0             192.168.2.222:46786        192.168.2.220:1521
         cubic wscale:7,6 rto:201 rtt:1.875/0.75 ato:40 cwnd:10 send 61.8Mbps rcv_rtt:8 rcv_space:14600
ESTAB      0      0             192.168.2.222:46798        192.168.2.220:1521
         cubic wscale:7,6 rto:201 rtt:1.875/0.75 ato:40 cwnd:10 send 61.8Mbps rcv_rtt:12 rcv_space:14600
ESTAB      0      0             192.168.2.222:46819        192.168.2.220:1521
         cubic wscale:7,6 rto:201 rtt:1.875/0.75 ato:40 cwnd:10 send 61.8Mbps rcv_rtt:9 rcv_space:14600
ESTAB      0      0      ::ffff:192.168.2.222:46674 ::ffff:192.168.2.220:1521
         cubic wscale:7,6 rto:206 rtt:6.75/8.25 ato:40 cwnd:10 send 17.2Mbps rcv_rtt:2.125 rcv_space:14600
[root@222-comecs ~]# watch -n1 -d "ss -nmp"
Every 1.0s: ss -nmp                                                                                                                   Mon Dec 22 12:00:03 2014

State      Recv-Q Send-Q        Local Address:Port          Peer Address:Port
ESTAB      0      0             192.168.2.222:46839        192.168.2.220:1521   users:(("asterisk",30976,157))
         mem:(r0,w0,f0,t0)
ESTAB      0      0             192.168.2.222:46797        192.168.2.220:1521   users:(("asterisk",30976,73))
         mem:(r0,w0,f0,t0)
ESTAB      0      0             192.168.2.222:46786        192.168.2.220:1521   users:(("asterisk",30976,51))
         mem:(r0,w0,f0,t0)
ESTAB      0      0             192.168.2.222:46798        192.168.2.220:1521   users:(("asterisk",30976,75))
         mem:(r0,w0,f0,t0)
ESTAB      0      0             192.168.2.222:46819        192.168.2.220:1521   users:(("asterisk",30976,117))
         mem:(r0,w0,f0,t0)
ESTAB      0      0      ::ffff:192.168.2.222:46674 ::ffff:192.168.2.220:1521   users:(("java",28435,36))
         mem:(r0,w0,f0,t0)

[root@222-comecs ~]# watch -n1 -d "ss -np"
Every 1.0s: ss -np                                                                                                                    Mon Dec 22 11:59:16 2014

State      Recv-Q Send-Q        Local Address:Port          Peer Address:Port
ESTAB      0      0             192.168.2.222:46839        192.168.2.220:1521   users:(("asterisk",30976,157))
ESTAB      0      0             192.168.2.222:46797        192.168.2.220:1521   users:(("asterisk",30976,73))
ESTAB      0      0             192.168.2.222:46786        192.168.2.220:1521   users:(("asterisk",30976,51))
ESTAB      0      0             192.168.2.222:46798        192.168.2.220:1521   users:(("asterisk",30976,75))
ESTAB      0      0             192.168.2.222:46819        192.168.2.220:1521   users:(("asterisk",30976,117))
ESTAB      0      0      ::ffff:192.168.2.222:46674 ::ffff:192.168.2.220:1521   users:(("java",28435,36))
ESTAB      0      0             192.168.2.222:46809        192.168.2.220:1521   users:(("asterisk",30976,97))
ESTAB      0      0             192.168.2.222:46789        192.168.2.220:1521   users:(("asterisk",30976,57))
ESTAB      0      0             192.168.2.222:41986        192.168.2.220:1521   users:(("java",28345,21))
ESTAB      0      0             192.168.2.222:46812        192.168.2.220:1521   users:(("asterisk",30976,103))
ESTAB      0      0             192.168.2.222:10000        223.104.11.85:47208  users:(("asterisk",30976,18))
ESTAB      0      0             192.168.2.222:46785        192.168.2.220:1521   users:(("asterisk",30976,49))
ESTAB      0      0             192.168.2.222:46784        192.168.2.220:1521   users:(("asterisk",30976,47))
ESTAB      0      0             192.168.2.222:49205        192.168.2.220:1521   users:(("httpd",15354,16))
[root@222-comecs ~]# watch -n1 -d "ss -ne"
Every 1.0s: ss -ne                                                                                                                    Mon Dec 22 12:02:04 2014

State      Recv-Q Send-Q        Local Address:Port          Peer Address:Port
ESTAB      0      0             192.168.2.222:46839        192.168.2.220:1521   ino:5102290 sk:ffff8800375a58c0
ESTAB      0      0             192.168.2.222:46797        192.168.2.220:1521   ino:5102205 sk:ffff88003df53640
ESTAB      0      0             192.168.2.222:46786        192.168.2.220:1521   ino:5102183 sk:ffff88003ca95040
ESTAB      0      0             192.168.2.222:46798        192.168.2.220:1521   ino:5102207 sk:ffff88003d230ac0
ESTAB      0      0             192.168.2.222:46819        192.168.2.220:1521   ino:5102249 sk:ffff88003d697780
ESTAB      0      0      ::ffff:192.168.2.222:46674 ::ffff:192.168.2.220:1521   ino:4463116 sk:ffff88003b55f040
ESTAB      0      0             192.168.2.222:46809        192.168.2.220:1521   ino:5102229 sk:ffff88003e39a880
ESTAB      0      0             192.168.2.222:46789        192.168.2.220:1521   ino:5102189 sk:ffff88000ab48740
ESTAB      0      0             192.168.2.222:41986        192.168.2.220:1521   ino:4057728 sk:ffff88003ca942c0
ESTAB      0      0             192.168.2.222:46812        192.168.2.220:1521   ino:5102235 sk:ffff88003d579040
ESTAB      0      0             192.168.2.222:10000        223.104.11.85:47208  timer:(keepalive,2min50sec,0) ino:9738746 sk:ffff88003b6d1000
ESTAB      0      0             192.168.2.222:46785        192.168.2.220:1521   ino:5102181 sk:ffff88003ca94980
ESTAB      0      0             192.168.2.222:46784        192.168.2.220:1521   ino:5102179 sk:ffff88003ca95700
ESTAB      0      0             192.168.2.222:49205        192.168.2.220:1521   uid:48 ino:9911907 sk:ffff88003b4c11c0