Apache访问日志分析脚本【第二版】
#!/usr/bin/env python # coding=utf-8 #------------------------------------------------------ # Name: Apache 日志分析脚本 # Purpose: 此脚本只用来分析Apache的访问日志 # Version: 2.0 # Author: LEO # BLOG: http://linux5588.blog.51cto.com # EMAIL: chanyipiaomiao@163.com # Created: 2013-4-26 # Modified: 2013-5-4 # Copyright: (c) LEO 2013 #------------------------------------------------------ import sys import time #该类是用来打印格式 class displayFormat(object): def format_size(self,size): '''''格式化流量单位''' KB = 1024 MB = 1048576 GB = 1073741824 TB = 1099511627776 if size >= TB : size = str(size / TB) + 'T' elif size < KB : size = str(size) + 'B' elif size >= GB and size < TB: size = str(size / GB) + 'G' elif size >= MB and size < GB : size = str(size / MB) + 'M' else : size = str(size / KB) + 'K' return size formatstring = '%-15s %-10s %-12s %8s %10s %10s %10s %10s %10s %10s %10s' def transverse_line(self) : '''''输出横线''' print self.formatstring % ('-'*15,'-'*10,'-'*12,'-'*12,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10) def head(self): '''''输出头部信息''' print self.formatstring % ('IP','Traffic','Times','Times%','200','404','500','403','302','304','503') def error_print(self) : '''''输出错误信息''' print print 'Usage : ' + sys.argv[0] + ' ApacheLogFilePath [Number]' print sys.exit(1) def execut_time(self): '''''输出脚本执行的时间''' print print "Script Execution Time: %.3f second" % time.clock() print #该类是用来生成主机信息的字典 class hostInfo(object): host_info = ['200','404','500','302','304','503','403','times','size'] def __init__(self,host): self.host = host = {}.fromkeys(self.host_info,0) def increment(self,status_times_size,is_size): '''''该方法是用来给host_info中的各个值加1''' if status_times_size == 'times': self.host['times'] += 1 elif is_size: self.host['size'] = self.host['size'] + status_times_size else: self.host[status_times_size] += 1 def get_value(self,value): '''''该方法是取到各个主机信息中对应的值''' return self.host[value] #该类是用来分析文件 class fileAnalysis(object): def __init__(self): '''''初始化一个空字典''' self.report_dict = {} self.total_request_times,self.total_traffic,self.total_200, \ self.total_404,self.total_500,self.total_403,self.total_302, \ self.total_304,self.total_503 = 0,0,0,0,0,0,0,0,0 def split_eachline_todict(self,line): '''''分割文件中的每一行，并返回一个字典''' split_line = line.split() split_dict = {'remote_host':split_line[0],'status':split_line[-2],'bytes_sent':split_line[-1],} return split_dict def generate_log_report(self,logfile): '''''读取文件，分析split_eachline_todict方法生成的字典''' for line in logfile: try: line_dict = self.split_eachline_todict(line) host = line_dict['remote_host'] status = line_dict['status'] except ValueError : continue except IndexError : continue if host not in self.report_dict : host_info_obj = hostInfo(host) self.report_dict[host] = host_info_obj else : host_info_obj = self.report_dict[host] host_info_obj.increment('times',False) if status in host_info_obj.host_info : host_info_obj.increment(status,False) try: bytes_sent = int(line_dict['bytes_sent']) except ValueError: bytes_sent = 0 host_info_obj.increment(bytes_sent,True) return self.report_dict def return_sorted_list(self,true_dict): '''''计算各个状态次数、流量总量，请求的总次数，并且计算各个状态的总量 并生成一个正真的字典，方便排序''' for host_key in true_dict : host_value = true_dict[host_key] times = host_value.get_value('times') self.total_request_times = self.total_request_times + times size = host_value.get_value('size') self.total_traffic = self.total_traffic + size o200 = host_value.get_value('200') o404 = host_value.get_value('404') o500 = host_value.get_value('500') o403 = host_value.get_value('403') o302 = host_value.get_value('302') o304 = host_value.get_value('304') o503 = host_value.get_value('503') true_dict[host_key] = {'200':o200,'404':o404,'500':o500,'403':o403,'302':o302,'304':o304, \ '503':o503,'times':times,'size':size} self.total_200 = self.total_200 + o200 self.total_404 = self.total_404 + o404 self.total_500 = self.total_500 + o500 self.total_302 = self.total_302 + o302 self.total_304 = self.total_304 + o304 self.total_503 = self.total_503 + o503 sorted_list = sorted(true_dict.items(),key=lambda t:(t[1]['times'],t[1]['size']),reverse=True) return sorted_list class Main(object): def main(self) : '''''主调函数''' display_format = displayFormat() arg_length = len(sys.argv) if arg_length == 1 : display_format.error_print() elif arg_length == 2 or arg_length == 3: infile_name = sys.argv[1] try : infile = open(infile_name,'r') if arg_length == 3 : lines = int(sys.argv[2]) else : lines = 0 except IOError,e : print print e display_format.error_print() except ValueError : print print "Please Enter A Volid Number !!" display_format.error_print() else : display_format.error_print() fileAnalysis_obj = fileAnalysis() not_true_dict = fileAnalysis_obj.generate_log_report(infile) log_report = fileAnalysis_obj.return_sorted_list(not_true_dict) total_ip = len(log_report) if lines : log_report = log_report[0:lines] infile.close() print total_traffic = display_format.format_size(fileAnalysis_obj.total_traffic) total_request_times = fileAnalysis_obj.total_request_times print 'Total IP: %s Total Traffic: %s Total Request Times: %d' \ % (total_ip,total_traffic,total_request_times) print display_format.head() display_format.transverse_line() for host in log_report : times = host[1]['times'] times_percent = (float(times) / float(fileAnalysis_obj.total_request_times)) * 100 print display_format.formatstring % (host[0],\ display_format.format_size(host[1]['size']),\ times,str(times_percent)[0:5],\ host[1]['200'],host[1]['404'],\ host[1]['500'],host[1]['403'],\ host[1]['302'],host[1]['304'],host[1]['503']) if (not lines) or total_ip == lines : display_format.transverse_line() print display_format.formatstring % (total_ip,total_traffic, \ total_request_times,'100%',\ fileAnalysis_obj.total_200,\ fileAnalysis_obj.total_404,\ fileAnalysis_obj.total_500, \ fileAnalysis_obj.total_403,\ fileAnalysis_obj.total_302, \ fileAnalysis_obj.total_304,\ fileAnalysis_obj.total_503) display_format.execut_time() if __name__ == '__main__': main_obj = Main() main_obj.main()1,查看apache进程:
ps aux | grep httpd | grep -v grep | wc -l
2,查看80端口的tcp连接:
netstat -tan | grep "ESTABLISHED" | grep ":80" | wc -l
3,通过日志查看当天ip连接数，过滤重复:
cat access_log | grep "20/Oct/2008" | awk '{print $2}' | sort | uniq -c | sort -nr
4,当天ip连接数最高的ip都在干些什么(原来是蜘蛛):
cat access_log | grep "20/Oct/2008:00" | grep "122.102.7.212" | awk '{print $8}' | sort | uniq -c | sort -nr | head -n 10
5,当天访问页面排前10的url:
cat access_log | grep "20/Oct/2008:00" | awk '{print $8}' | sort | uniq -c | sort -nr | head -n 10
6,用tcpdump嗅探80端口的访问看看谁最高
tcpdump -i eth0 -tnn dst port 80 -c 1000 | awk -F"." '{print $1"."$2"."$3"."$4}' | sort | uniq -c | sort -nr
接着从日志里查看该ip在干嘛:
cat access_log | grep 122.102.7.212| awk '{print $1"\t"$8}' | sort | uniq -c | sort -nr | less
7,查看某一时间段的ip连接数:
grep "2006:0[7-8]" www20060723.log | awk '{print $2}' | sort | uniq -c| sort -nr | wc -l
==============================nginx
log_format main '[$time_local] $remote_addr $status $request_time $body_bytes_sent "$request" "$http_referer"';
access_log /data0/logs/access.log main;
格式如下：
[21/Mar/2011:11:52:15 +0800] 58.60.188.61 200 0.265 28 "POST /event/time HTTP/1.1" "http://host/loupan/207846/feature"
通过日志查看当天ip连接数，过滤重复
cat access.log | grep "20/Mar/2011" | awk '{print $3}' | sort | uniq -c | sort -nr
38 112.97.192.16
20 117.136.31.145
19 112.97.192.31
3 61.156.31.20
2 209.213.40.6
1 222.76.85.28
当天访问页面排前10的url:
cat access.log | grep "20/Mar/2011" | awk '{print $8}' | sort | uniq -c | sort -nr | head -n 10
找出访问次数最多的10个IP
awk '{print $3}' access.log |sort |uniq -c|sort -nr|head
10680 10.0.21.17
1702 10.0.20.167
823 10.0.20.51
504 10.0.20.255
215 58.60.188.61
192 183.17.161.216
38 112.97.192.16
20 117.136.31.145
19 112.97.192.31
6 113.106.88.10
找出某天访问次数最多的10个IP
cat /tmp/access.log | grep "20/Mar/2011" |awk '{print $3}'|sort |uniq -c|sort -nr|head
38 112.97.192.16
20 117.136.31.145
19 112.97.192.31
3 61.156.31.20
2 209.213.40.6
1 222.76.85.28
当天ip连接数最高的ip都在干些什么:
cat access.log | grep "10.0.21.17" | awk '{print $8}' | sort | uniq -c | sort -nr | head -n 10
224 /test/themes/default/img/logo_index.gif
224 /test/themes/default/img/bg_index_head.jpg
224 /test/themes/default/img/bg_index.gif
219 /test/vc.php
219 /
213 /misc/js/global.js
211 /misc/jsext/popup.ext.js
211 /misc/js/common.js
210 /sladmin/home
197 /misc/js/flib.js
找出访问次数最多的几个分钟
awk '{print $1}' access.log | grep "20/Mar/2011" |cut -c 14-18|sort|uniq -c|sort -nr|head
24 16:49
19 16:17
16 16:51
11 16:48
4 16:50
3 16:52
1 20:09
1 20:05
1 20:03
1 19:55
