时间：2021-07-01 10:21:17 帮助过：4人阅读
#!/usr/bin/python# -*- coding: utf-8 -*-"""用于切分 nginx日志nginx日志 格式：log_format access '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" $http_x_forwarded_for';INSERT INTO `nginxlog` (`status`, `remote_user`, `http_referer`, `remote_addr`, `http_x_forwarded_for`, `hostname`, `request`, `request_type`, `http_user_agent`, `time_local`) VALUES ('2', '2', '2', '2', '2', '2', '2', '2', '2', '2')table sql:CREATE TABLE `nginxlog` ( `status` int(4) DEFAULT NULL, `remote_user` varchar(20) DEFAULT NULL, `http_referer` text, `remote_addr` varchar(20) DEFAULT NULL, `http_x_forwarded_for` varchar(20) DEFAULT NULL, `hostname` varchar(50) DEFAULT NULL, `request` varchar(200) DEFAULT NULL, `request_type` varchar(10) DEFAULT NULL, `http_user_agent` varchar(200) DEFAULT NULL, `time_local` int(10) unsigned DEFAULT NULL, `server_id` int(5) unsigned DEFAULT NULL) ENGINE=MyISAM DEFAULT CHARSET=utf8"""#--------------------------config----------------------------------------------------------------------#configmysql = {}mysql['mhost'] = 'localhost'mysql['muser'] = 'root'mysql['mpwd'] = ''mysql['mport'] = '3306'mysql['mdbname'] = 'nginxlog' #获取当前执行目录 工作目录ROOT = '/home/nginxweblog/app/'#ftp 目录 用于存放生产 原始 日志Sourcedatadir = '/home/nginxweblog/sourcedata/'#log runLogfile = ROOT+'nginxlog.log'#--------------------------------import--------------------------------------------------------import re,time,osimport MySQLdb#--------------------------------Sublog--------------------------------------------------------#分析 每行的 nginxlog 数据class Sublog: def __init__(self, logline=None): if logline: self.logline = logline.strip() self.logdate = {} #self.run() def run(self): if self.logline: self.splitlog() self.getlog() def splitlog(self): # 按照空格切分日志 self.splitlogd = re.split(' ', self.logline) #print self.splitlogd return self.splitlogd def getlog(self): if len(self.splitlogd) >= 20: self.logdate['remote_addr'] = self.splitlogd[0] self.logdate['remote_user'] = self.splitlogd[2] self.logdate['time_local'] = time.mktime(time.strptime(self.splitlogd[3], '[%d/%b/%Y:%H:%M:%S')) self.logdate['request_type'] = self.splitlogd[5].replace('/"', '') self.logdate['request'] = self.splitlogd[6] self.logdate['status'] = self.splitlogd[8] self.logdate['http_referer'] = MySQLdb.escape_string(self.splitlogd[10].replace('/"', '')) self.logdate['http_x_forwarded_for'] = self.splitlogd[len(self.splitlogd)-1] self.get_http_user_agent() self.get_hostname() def get_http_user_agent(self): agent = re.compile(r'/"(.*?)/"') self.logdate['http_user_agent'] = MySQLdb.escape_string(agent.findall(self.logline)[2]) def get_hostname(self): hostname_r = re.compile(r'http://(.*?)/') ishostname = hostname_r.findall(self.splitlogd[10]) #print ishostname if len(ishostname) >= 1: self.logdate['hostname'] = ishostname[0] else: self.logdate['hostname'] = '-' #--------------------------------insert_log---------------------------------------------# 生成 csv 数据 ||| 分割 数据 ， 主要用于 mysql快速导入class insert_log(Sublog): def __init__(self, logpath, serverid): Sublog.__init__(self) self.nowtime = time.strftime("%Y_%m_%d",time.localtime(time.time())) self.sunlogfile = "nginxlog_"+self.nowtime+'.csv' self.logpath = logpath self.serverid = serverid self.numbres = 0 #self.Mysql_db_instance = Mysql_db() def createsubfile(self): self.opencsvfile() self.subfile = open(ROOT+'CSV/'+self.sunlogfile, 'a') olog = open(self.logpath) for i in olog: #self.logdate = {} self.logline = i.strip() self.run() if self.logdate: #print self.logdate self.createsql() #self.insertdb() #time.sleep(3) self.numbres = self.numbres + 1 self.subfile.close() return [ROOT+'CSV/'+self.sunlogfile, self.sunlogfile, self.numbres] def insertdb(self): sql = "INSERT INTO `nginxlog` (`status`, `remote_user`, `http_referer`, `remote_addr`, `http_x_forwarded_for`, `hostname`, `request`, `request_type`, `http_user_agent`, `time_local`) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s');" % (self.logdate['status'], self.logdate['remote_user'], self.logdate['http_referer'], self.logdate['remote_addr'], self.logdate['http_x_forwarded_for'], self.logdate['hostname'], self.logdate['request'], self.logdate['request_type'], self.logdate['http_user_agent'], self.logdate['time_local']) #print sql self.subfile.write(sql + '/n') #self.Mysql_db_instance.insertsql(sql) #self.Mysql_db_instance.commit() def createsql(self): c = "%s|||%s|||%s|||%s|||%s|||%s|||%s|||%s|||%s|||%s|||%s" % (self.logdate['status'], self.logdate['remote_user'], self.logdate['http_referer'], self.logdate['remote_addr'], self.logdate['http_x_forwarded_for'], self.logdate['hostname'], self.logdate['request'], self.logdate['request_type'], self.logdate['http_user_agent'], self.logdate['time_local'],self.serverid) self.subfile.write(c + '/n') def opencsvfile(self): if os.path.isdir(ROOT+'CSV'): pass else: os.mkdir(ROOT+'CSV')#--------------------------------function ---------------------------------------------def load_mysql(csvfile): if mysql['mpwd']: os.system("""mysql -h%s -u%s -p%s -e "LOAD DATA INFILE '%s' INTO TABLE nginxlog.nginxlog FIELDS TERMINATED BY '|||';" && rm -rf %s """ % (mysql['mhost'], mysql['muser'], mysql['mpwd'], csvfile, csvfile)) else: os.system("""mysql -h%s -u%s -e "LOAD DATA INFILE '%s' INTO TABLE nginxlog.nginxlog FIELDS TERMINATED BY '|||';" && rm -rf %s """ % (mysql['mhost'], mysql['muser'], csvfile, csvfile)) #索引处理 type = 0 删除索引， 1创建索引def mysqlindex(type): if mysql['mpwd']: ism = "mysql -h%s -u%s -p%s -e " % (mysql['mhost'], mysql['muser'], mysql['mpwd']) else: ism = "mysql -h%s -u%s -e " % (mysql['mhost'], mysql['muser']) if type == 0: os.system(" %s 'alter table nginxlog.nginxlog drop index time_local' " % ism) os.system(" %s 'alter table nginxlog.nginxlog drop index hostname' " % ism) os.system(" %s 'alter table nginxlog.nginxlog drop index remote_addr' " % ism) elif type == 1: os.system(" %s 'alter table nginxlog.nginxlog add index time_local(time_local)' " % ism) os.system(" %s 'alter table nginxlog.nginxlog add index hostname(hostname)' " % ism) os.system(" %s 'alter table nginxlog.nginxlog add index remote_addr(remote_addr)' " % ism)""" 日志记录函数 """def write_logs(logconten): logfile_path = Logfile if logfile_path and logconten: log_write = open(logfile_path, 'a') log_write.write(logconten+'/n') log_write.close()##--------------------------------运行---------------------------------------------def run(): list = os.listdir(Sourcedatadir) print 'start.......' write_logs('-----------%s start.....----%s-----------------' % ( time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time())), str(list)) ) for line in list: if os.path.isfile(Sourcedatadir+line): s = re.split('_', line) if re.match(r'^/d.*$', s[0]): write_logs('time(%s) logfile(%s) start ....' % (time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time())), line)) insert_log_instance = insert_log(Sourcedatadir+line, s[0]) gfiledata = insert_log_instance.createsubfile() load_mysql(gfiledata[0]) os.remove(Sourcedatadir+line) write_logs('time(%s) logfile(%s) log numbers(%s) stop ....' % (time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time())), line, str(gfiledata[2])) ) print Sourcedatadir+line+ ' OK ....... ' #-------------------------------- exece ---------------------------------------------if __name__ == "__main__": #run_inotify(Sourcedatadir, run) run()