日志分析!
对最近五分钟的日志进行分析,取出访问最多的IP,URL,以及五分钟内的访问流量
#!/bin/sh name=`cat /etc/salt/minion | grep "^id" | awk '{print $2}'` ipaddr=`/sbin/ifconfig | grep "inet addr" | egrep -v "10\.|127\." | awk -F'[: ]+' '{print $4}'` #LogPath='/usr/local/tengine/logs/' ##日志的时间格式。 function Time(){ FileTime=`head -1 $Logfile |awk -F"[[ / :]" '{print "["$5"/"$6"/"$7}'` start_time=${FileTime}:`date -d "6 mins ago" +%H:%M` #now_time=${FileTime}:`date +%H:%M` } ##取得指定最新时间内的日志,并进行分析 function IpUrlFlow(){ Time path="/tmp/backup/log" [ ! -d $path/flow/ ] && mkdir -p $path/flow file="$path/${Logfile}.time.log" FileIp="$path/${Logfile}.IP.log" FileUri="$path/${Logfile}.uri.sort.log" FileIpStat="$path/${Logfile}.flow.sort.log" FileStat="$path/flow/${Logfile}.flow.log" #取出直接时间内的日志 #awk -v start_time=$start_time -v now_time=$now_time '{if ($4>start_time && $4<now_time) print $0 }' ${Logfile} > $file awk -v start_time=$start_time -v now_time=$now_time '{if ($4>start_time) print $0 }' ${Logfile} > $file if [ -s $file ] then #对取出的日志进行统计,取出访问次数最多的ip以及次数 awk '{S[$1]++}END{for(n in S) print S[n]" "n }' $file |sort -rn |head -20 > ${FileIp} #awk '{S[$1]++}END{for(n in S) print S[n]" "n }' $file |sort -rn |head -20 > ${FileIpStat} #对取出的日志进行统计,取出访问次数最多的URI以及次数 awk '{S[$7]++}END{for(n in S) print S[n]" "n }' $file |sort -rn |head -20 > ${FileUri} #awk '{S[$7]++}END{for(n in S) print S[n]" "n }' $file |sort -rn |head -20 >> ${FileIpStat} #对取出的日志进行统计,取出访问流量最多的ip以及流量大小,单位M awk '{S[$1]=+$10}END{for(n in S) print S[n]" "n }' $file |sort -rn|head -20 |awk '{a=$1/1024/1014}{print a " M "$2}' >${FileIpStat} #awk '{S[$1]=+$10}END{for(n in S) print S[n]" "n }' $file |sort -rn|head -20 |awk '{a=$1/1024/1014}{print a " M "$2}' >>${FileIpStat} #整合到一个文件! echo -e "IP_start" > ${FileStat} cat ${FileIp} >> ${FileStat} echo -e "IP_end\n" >> ${FileStat} echo -e "URL_start" >> ${FileStat} cat ${FileUri} >> ${FileStat} echo -e "URL_end\n" >> ${FileStat} echo -e "Flow_star" >> ${FileStat} cat ${FileIpStat} >> ${FileStat} echo -e "Flow_end\n" >> ${FileStat} fi } #Ip function main(){ cd ${LogPath} for Logfile in `ls acce*` do if [ ! -s ${Logfile} ] then echo ${Logfile} else Ip fi done } main ############################################################################### ############################################################################### ############################################################################### #!/bin/sh name=`cat /etc/salt/minion | grep "^id" | awk '{print $2}'` ipaddr=`/sbin/ifconfig | grep "inet addr" | egrep -v "10\.|127\." | awk -F'[: ]+' '{print $4}'` LogPath='/usr/local/tengine/logs/' #filelog='access_www.log' #Logfile="${LogPath}${filelog}" function Time(){ FileTime=`head -1 $Logfile |awk -F"[[ / :]" '{print "["$5"/"$6"/"$7}'` start_time=${FileTime}:`date -d "6 mins ago" +%H:%M` now_time=${FileTime}:`date +%H:%M` #echo $FileTime $start_time $now_time } function Ip(){ Time path="/script/shell/gaogd/LogAnalyze/log" [ ! -d $path/flow ] && mkdir -p $path/flow file="$path/${Logfile}.time.log" #FileIp="$path/${Logfile}.IP.log" #FileUri="$path/${Logfile}.uri.sort.log" #FileIpStat="$path/${Logfile}.stat.sort.log" FileStat="$path/flow/${Logfile}.Statistics.log" awk -v start_time=$start_time -v now_time=$now_time '{if ($4>start_time && $4<now_time) print $0 }' ${Logfile} > $file if [ -s $file ] then echo -e "IP_start" > ${FileStat} awk '{S[$1]++}END{for(n in S) print S[n]" "n }' $file |sort -rn |head -20 >> ${FileStat} echo -e "IP_end\n" >> ${FileStat} echo -e "URL_start" >> ${FileStat} awk '{S[$7]++}END{for(n in S) print S[n]" "n }' $file |sort -rn |head -20 >> ${FileStat} echo -e "URL_end\n" >> ${FileStat} echo -e "Flow_star" >> ${FileStat} awk '{S[$1]=+$10}END{for(n in S) print S[n]" "n }' $file |sort -rn|head -20 |awk '{a=$1/1024/1014}{print a " M "$2}' >>${FileStat} echo -e "Flow_end\n" >> ${FileStat} else if [ -s ${FileStat} ] then mv ${FileStat} /tmp fi fi } #Ip function main(){ cd ${LogPath} for Logfile in `ls acce*` do if [ ! -s ${Logfile} ] then echo ${Logfile} else Ip fi done } main