python分析nignx访问日志脚本分享

#!/usr/bin/env python
# coding=utf-8

#------------------------------------------------------
# Name:     nginx 日志分析脚本
# Purpose:   此脚本只用来分析nginx的访问日志
# Version:   1.0
# Author:    LEO
# Created:   2013-05-07
# Modified:   2013-05-07
# Copyright:  (c) LEO 2013
#------------------------------------------------------

import sys
import time

#该类是用来打印格式
class displayFormat(object):

def format_size(self,size):
   '''''格式化流量单位'''
   KB = 1024      #KB -> B B是字节
   MB = 1048576    #MB -> B
   GB = 1073741824   #GB -> B
   TB = 1099511627776 #TB -> B
   if size >= TB :
     size = str(size / TB) + 'T'
   elif size < KB :
     size = str(size) + 'B'
   elif size >= GB and size < TB:
     size = str(size / GB) + 'G'
   elif size >= MB and size < GB :
     size = str(size / MB) + 'M'
   else :
     size = str(size / KB) + 'K'
   return size

#定义字符串格式化
 formatstring = '％-15s ％-10s ％-12s ％8s ％10s ％10s ％10s ％10s ％10s ％10s ％10s'

def transverse_line(self) :
   '''''输出横线'''
   print self.formatstring ％ ('-'*15,'-'*10,'-'*12,'-'*12,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10)

def head(self):
   '''''输出头部信息'''
   print self.formatstring ％ ('IP','Traffic','Times','Times％','200','404','500','403','302','304','503')

def error_print(self) :
   '''''输出错误信息'''
   print
   print 'Usage : ' + sys.argv[0] + ' NginxLogFilePath [Number]'
   print
   sys.exit(1)

def execut_time(self):
   '''''输出脚本执行的时间'''
   print
   print "Script Execution Time: ％.3f second" ％ time.clock()
   print

#该类是用来生成主机信息的字典
class hostInfo(object):
 host_info = ['200','404','500','302','304','503','403','times','size']

def __init__(self,host):
   self.host = host = {}.fromkeys(self.host_info,0)

def increment(self,status_times_size,is_size):
   '''''该方法是用来给host_info中的各个值加1'''
   if status_times_size == 'times':
     self.host['times'] += 1
   elif is_size:
     self.host['size'] = self.host['size'] + status_times_size
   else:
     self.host[status_times_size] += 1

def get_value(self,value):
   '''''该方法是取到各个主机信息中对应的值'''
   return self.host[value]

#该类是用来分析文件
class fileAnalysis(object):
 def __init__(self):
   '''''初始化一个空字典'''
   self.report_dict = {}
   self.total_request_times,self.total_traffic,self.total_200,
   self.total_404,self.total_500,self.total_403,self.total_302,
   self.total_304,self.total_503 = 0,0,0,0,0,0,0,0,0

def split_eachline_todict(self,line):
   '''''分割文件中的每一行，并返回一个字典'''
   split_line = line.split()
   split_dict = {'remote_host':split_line[0],'status':split_line[8],
          'bytes_sent':split_line[9],}
   return split_dict

def generate_log_report(self,logfile):
   '''''读取文件，分析split_eachline_todict方法生成的字典'''
   for line in logfile:
     try:
       line_dict = self.split_eachline_todict(line)
       host = line_dict['remote_host']
       status = line_dict['status']
     except ValueError :
       continue
     except IndexError :
       continue

if host not in self.report_dict :
       host_info_obj = hostInfo(host)
       self.report_dict[host] = host_info_obj
     else :
       host_info_obj = self.report_dict[host]

host_info_obj.increment('times',False)
     if status in host_info_obj.host_info :
       host_info_obj.increment(status,False)
     try:
       bytes_sent = int(line_dict['bytes_sent'])
     except ValueError:
       bytes_sent = 0
     host_info_obj.increment(bytes_sent,True)
   return self.report_dict

def return_sorted_list(self,true_dict):
   '''''计算各个状态次数、流量总量，请求的总次数，并且计算各个状态的总量 并生成一个正真的字典，方便排序'''
   for host_key in true_dict :
     host_value = true_dict[host_key]
     times = host_value.get_value('times')            
     self.total_request_times = self.total_request_times + times
     size = host_value.get_value('size')            
     self.total_traffic = self.total_traffic + size  

o200 = host_value.get_value('200')
     o404 = host_value.get_value('404')
     o500 = host_value.get_value('500')
     o403 = host_value.get_value('403')
     o302 = host_value.get_value('302')
     o304 = host_value.get_value('304')
     o503 = host_value.get_value('503')

true_dict[host_key] = {'200':o200,'404':o404,'500':o500,
                 '403':o403,'302':o302,'304':o304,
                 '503':o503,'times':times,'size':size}

self.total_200 = self.total_200 + o200
     self.total_404 = self.total_404 + o404
     self.total_500 = self.total_500 + o500
     self.total_302 = self.total_302 + o302
     self.total_304 = self.total_304 + o304
     self.total_503 = self.total_503 + o503

sorted_list = sorted(true_dict.items(),key=lambda t:(t[1]['times'],
                              t[1]['size']),reverse=True)

return sorted_list

class Main(object):
 def main(self) :
   '''''主调函数'''
   display_format = displayFormat()
   arg_length = len(sys.argv)
   if arg_length == 1 :
     display_format.error_print()
   elif arg_length == 2 or arg_length == 3:
     infile_name = sys.argv[1]
     try :
       infile = open(infile_name,'r')
       if arg_length == 3 :
         lines = int(sys.argv[2])
       else :
         lines = 0
     except IOError,e :
       print
       print e
       display_format.error_print()
     except ValueError :
       print
       print "Please Enter A Volid Number !!"
       display_format.error_print()
   else :
     display_format.error_print()

fileAnalysis_obj = fileAnalysis()
   not_true_dict = fileAnalysis_obj.generate_log_report(infile)
   log_report = fileAnalysis_obj.return_sorted_list(not_true_dict)
   total_ip = len(log_report)
   if lines :
     log_report = log_report[0:lines]
   infile.close()

print
   total_traffic = display_format.format_size(fileAnalysis_obj.total_traffic)
   total_request_times = fileAnalysis_obj.total_request_times
   print 'Total IP: ％s  Total Traffic: ％s  Total Request Times: ％d'
      ％ (total_ip,total_traffic,total_request_times)
   print
   display_format.head()
   display_format.transverse_line()

for host in log_report :
     times = host[1]['times']
     times_percent = (float(times) / float(fileAnalysis_obj.total_request_times)) * 100
     print display_format.formatstring ％ (host[0],
                        display_format.format_size(host[1]['size']),
                        times,str(times_percent)[0:5],
                        host[1]['200'],host[1]['404'],
                        host[1]['500'],host[1]['403'],
                        host[1]['302'],host[1]['304'],host[1]['503'])

if (not lines) or total_ip == lines :
     display_format.transverse_line()
     print display_format.formatstring ％ (total_ip,total_traffic,
                        total_request_times,'100％',
                        fileAnalysis_obj.total_200,
                        fileAnalysis_obj.total_404,
                        fileAnalysis_obj.total_500,
                        fileAnalysis_obj.total_403,
                        fileAnalysis_obj.total_302,
                        fileAnalysis_obj.total_304,
                        fileAnalysis_obj.total_503)

display_format.execut_time()

if __name__ == '__main__':
 main_obj = Main()
 main_obj.main()