OpenResty(Nginx Lua)统计网站访问信息

背景

之前的一篇文章openresty(nginx lua)统计域名状态码、平均响应时间和流量实现了对域名状态码，平均响应时间和流量的统计。但之前的统计方法没有实现当某一域名404或500等状态码超过一定数量后发送具体的url来快速定位位置。这个功能我们其实是通过统计网站日志来实现了。为了摆脱对网站日志的依赖以及提高统计性能，我们尝试把此功能也用nginx lua来实现。具体的使用方法与之前的文章一样，这里只是更新了两个lua脚本。

使用方法

1、获取域名www.centos.bz 404状态码数量

curl -s "localhost/domain_status?count=status&host=www.centos.bz&status=404"

输出:
10 688
第一列为状态码数量，第二列为域名请求总数
2、获取当域名www.centos.bz 404状态码超过50个时,输出前10个url

curl -s "localhost/domain_status?count=statusUrl&host=www.centos.bz&status=404&exceed=50&output=10"

输出:
/hello-world 90
/centos 10
第一列为url，第二列为url请求次数。
3、获取域名www.centos.bz upstream一分钟内平均耗时

curl -s "localhost/domain_status?count=upT&host=www.centos.bz"

输出:
0.02 452
第一列为upstream平均耗时，第二列为域名总请求次数。
4、获取当域名www.centos.bz upstream平均耗时超过0.5秒时,输出其url

curl -s "localhost/domain_status?count=upTUrl&host=www.centos.bz&exceed=0.5"

输出:
/hello.php 0.82 52
第一列为url，第二列为此url平均耗时,第三列为此url请求次数。监控此接口数据可以快速定位出具体哪些url慢了。
5、获取域名www.centos.bz request time平均耗时

curl -s "localhost/domain_status?count=reqT&host=www.centos.bz"

输出:
1.82 52
第一列为平均耗时，第二列为域名请求数。request time是指完成整个请求所需要的时间（包括把数据传输到用户浏览器的时间）。对于php请求，upstream time指的是nginx把php请求传给fastcgi到完成数据接收所需时间。所以request time永远大于upstream time。
6、获取域名www.centos.bz占用的带宽(单位:字节/秒)

curl -s "localhost/domain_status?count=flow&host=www.centos.bz"

输出:
1024 52
第一列为此域名一分钟内平均传输速率，单位为字节/秒，第二列为域名请求总数。

相关脚本

log_acesss.lua

local access = ngx.shared.access
local host = ngx.var.host or "unknow"
local status = ngx.var.status
local body_bytes_sent = ngx.var.body_bytes_sent
local request_time = ngx.var.request_time
local upstream_response_time = ngx.var.upstream_response_time or 0
local request_uri = ngx.var.request_uri or "/unknow"
local timestamp = ngx.time()
local expire_time = 70
 
local status_key = table.concat({host,"-",status,"-",timestamp})
local flow_key = table.concat({host,"-flow-",timestamp})
local req_time_key = table.concat({host,"-reqt-",timestamp})
local up_time_key = table.concat({host,"-upt-",timestamp})
local total_key = table.concat({host,"-total-",timestamp})
 
-- 域名总请求数
local n,e = access:incr(total_key,1)
if not n then
	access:set(total_key, 1, expire_time)
end
 
-- 域名状态码请求数
local n,e = access:incr(status_key,1)
if not n then
	access:set(status_key, 1, expire_time)
end
 
-- 域名流量
local n,e = access:incr(flow_key,body_bytes_sent)
if not n then
	access:set(flow_key, body_bytes_sent, expire_time)
end
 
-- 域名请求耗时
local n,e = access:incr(req_time_key,request_time)
if not n then
	access:set(req_time_key, request_time, expire_time)
end
 
-- 域名upstream耗时
local n,e = access:incr(up_time_key,upstream_response_time)
if not n then
	access:set(up_time_key, upstream_response_time, expire_time)
end
 
-- 获取不带参数的uri
local m, err = ngx.re.match(request_uri, "(.*?)?")
local request_without_args = m and m[1] or request_uri
 
-- 存储状态码大于400的url
if tonumber(status) >= 400 then
	-- 拼接url,状态码,字节数等字段
	local request_log_t = {}
	table.insert(request_log_t,host)
	table.insert(request_log_t,request_without_args)
	table.insert(request_log_t,status)
	local request_log = table.concat(request_log_t," ")
 
	-- 把拼接的字段储存在字典中
	local log_key = table.concat({"status-",timestamp})
	local request_log_dict = access:get(log_key) or ""
	if request_log_dict == "" then
		request_log_dict = request_log
	else	
		request_log_dict = table.concat({request_log_dict,"
",request_log})
	end
	access:set(log_key, request_log_dict, expire_time)
end
 
-- 存储upstream time大于0.5的url
if tonumber(upstream_response_time) > 0.5 then
	-- 拼接url,状态码,字节数等字段
	local request_log_t = {}
	table.insert(request_log_t,host)
	table.insert(request_log_t,request_without_args)
	table.insert(request_log_t,upstream_response_time)
	local request_log = table.concat(request_log_t," ")
 
	-- 把拼接的字段储存在字典中
	local log_key = table.concat({"upt-",timestamp})
	local request_log_dict = access:get(log_key) or ""
	if request_log_dict == "" then
		request_log_dict = request_log
	else	
		request_log_dict = table.concat({request_log_dict,"
",request_log})
	end
	access:set(log_key, request_log_dict, expire_time)
end

domain_status.lua

-- 各参数用法:
-- count=status,host=xxx.com,status=404 统计xxx.com域名一分钟内404状态码个数.
-- count=statusUrl,host=xxx.com,status=404,exceed=50,output=30 当xxx.com域名404状态码一分钟内超过50个时,输出前30个url，否则返回空.
 
-- count=upT,host=xxx.com 统计xxx.com域名一分钟内平均upsteam耗时
-- count=upTUrl,host=xxx.com,exceed=0.5 输出upstreamTime超过0.5秒的url,没有就返回空
 
-- count=reqT,host=xxx.com 统计xxx.com域名一分钟内平均请求耗时
-- count=flow,host=xxx.com 统计xxx.com域名一分钟内流量(单位字节/秒)
 
-- 函数: 获取迭代器值
local get_field = function(iterator)
    local m,err = iterator
    if err then
        ngx.log(ngx.ERR, "get_field iterator error: ", err)
        ngx.exit(ngx.HTTP_OK)
    end
    return m[0]
end
 
-- 函数: 按值排序table
local getKeysSortedByValue = function (tbl, sortFunction)
  local keys = {}
  for key in pairs(tbl) do
    table.insert(keys, key)
  end
 
  table.sort(keys, function(a, b)
    return sortFunction(tbl[a], tbl[b])
  end)
 
  return keys
end
 
-- 函数: 判断table是否存在某元素
local tbl_contain = function(table,element)
    for k in pairs(table) do
        if k == element then
            return true
        end
    end
    return false
end
 
local access = ngx.shared.access
local now = ngx.time()
local one_minute_ago = now - 60
 
-- 获取参数
local args = ngx.req.get_uri_args()
local count_arg = args["count"]
local host_arg = args["host"]
local status_arg = args["status"]
local exceed_arg = args["exceed"]
local output_arg = args["output"]
local count_t = {["status"]=0,["statusUrl"]=0,["upT"]=0,["upTUrl"]=0,["reqT"]=0,["flow"]=0}
 
-- 检查参数是否满足
if not tbl_contain(count_t,count_arg) then
    ngx.print("count arg invalid.")
    ngx.exit(ngx.HTTP_OK)
end
 
if not host_arg then
    ngx.print("host arg not found.")
    ngx.exit(ngx.HTTP_OK)
end
 
if count_arg == "status" and not status_arg then
    ngx.print("status arg not found.")
    ngx.exit(ngx.HTTP_OK)
end
 
if count_arg == "statusUrl" and not (status_arg and exceed_arg and output_arg)  then
    ngx.print("status or exceed or output arg not found.")
    ngx.exit(ngx.HTTP_OK)
end
 
if count_arg == "upTUrl" and not exceed_arg then
    ngx.print("exceed arg not found.")
    ngx.exit(ngx.HTTP_OK)
end
 
-- 检查参数是否合法
if status_arg and ngx.re.find(status_arg, "^[0-9]{3}$") == nil then
    ngx.print("status arg must be a valid httpd code.")
    ngx.exit(ngx.HTTP_OK)
end
 
if exceed_arg and ngx.re.find(exceed_arg, "^[0-9.]+$") == nil then
    ngx.print("exceed arg must be a number.")
    ngx.exit(ngx.HTTP_OK)
end
 
if output_arg and ngx.re.find(output_arg, "^[0-9]+$") == nil then
    ngx.print("output arg must be a number.")
    ngx.exit(ngx.HTTP_OK)
end
 
-- 开始统计
local url
local status_code
local upstream_time
local status_total = 0
local host
local req_total = 0
local flow_total = 0
local reqtime_total = 0
local upstream_total = 0
local status_url_t = {}
local upstream_url_t = {}
local upstream_url_count_t = {}
 
local status_log
local upt_log
 
for second_num=one_minute_ago,now do
    local flow_key = table.concat({host_arg,"-flow-",second_num})
    local req_time_key = table.concat({host_arg,"-reqt-",second_num})
    local up_time_key = table.concat({host_arg,"-upt-",second_num}) 
    local total_req_key = table.concat({host_arg,"-total-",second_num})
    local log_key
    local log_line
 
    -- 合并状态码大于等于400的请求日志到变量status_log
    log_key = table.concat({"status-",second_num})
    log_line = access:get(log_key) or ""
    if not (log_line == "") then
        status_log = table.concat({log_line,"
",status_log})
    end
 
    -- 合并upstream time大于0.5秒的请求日志到变量upt_log
    log_key = table.concat({"upt-",second_num})
    log_line = access:get(log_key) or ""
    if not (log_line == "") then
        upt_log = table.concat({log_line,"
",upt_log})
    end
 
    -- 域名总请求数
    local req_sum = access:get(total_req_key) or 0
    req_total = req_total + req_sum
 
    if count_arg == "status" or count_arg == "statusUrl" then
        local status_key = table.concat({host_arg,"-",status_arg,"-",second_num})
        local status_sum = access:get(status_key) or 0
        status_total = status_total + status_sum
    end
 
    if count_arg == "flow" then
        local flow_sum = access:get(flow_key) or 0
        flow_total = flow_total + flow_sum
    end
 
    if count_arg == "reqT" then
        local req_time_sum = access:get(req_time_key) or 0
        reqtime_total = reqtime_total + req_time_sum
    end
 
    if count_arg == "upT" then
        local up_time_sum = access:get(up_time_key) or 0
        upstream_total = upstream_total + up_time_sum
    end
end
 
-- 统计状态码url
if count_arg == "statusUrl" and status_log and not (status_log == "") then
    local iterator, err = ngx.re.gmatch(status_log,".+
")
    if not iterator then
        ngx.log(ngx.ERR, "status_log iterator error: ", err)
        return
    end
    for line in iterator do
        if not line[0] then
            ngx.log(ngx.ERR, "line[0] is nil")
            return
        end
        local iterator, err = ngx.re.gmatch(line[0],"[^ 
]+")
        if not iterator then
            ngx.log(ngx.ERR, "line[0] iterator error: ", err)
            return
        end
 
        host = get_field(iterator())
        url = get_field(iterator())
        status_code = get_field(iterator())
 
        if status_code == status_arg then
            if status_url_t[url] then
                status_url_t[url] = status_url_t[url] + 1
            else
                status_url_t[url] = 1
            end
        end
 
    end   
end
 
-- 统计upstream time大于0.5秒url
if count_arg == "upTUrl" and upt_log and not (upt_log == "") then
    local iterator, err = ngx.re.gmatch(upt_log,".+
")
    if not iterator then
        ngx.log(ngx.ERR, "upt_log iterator error: ", err)
        return
    end
    for line in iterator do
        if not line[0] then
            ngx.log(ngx.ERR, "line[0] is nil")
            return
        end
        local iterator, err = ngx.re.gmatch(line[0],"[^ 
]+")
        if not iterator then
            ngx.log(ngx.ERR, "line[0] iterator error: ", err)
            return
        end
 
        host = get_field(iterator())
        url = get_field(iterator())
        upstream_time = get_field(iterator())
        upstream_time = tonumber(upstream_time) or 0
 
        -- 统计各url upstream平均耗时
        if host == host_arg then
            if upstream_url_t[url] then
                upstream_url_t[url] = upstream_url_t[url] + upstream_time
            else
                upstream_url_t[url] = upstream_time
            end
 
            if upstream_url_count_t[url] then
                upstream_url_count_t[url] = upstream_url_count_t[url] + 1
            else
                upstream_url_count_t[url] = 1
            end
        end    
    end   
end
 
-- 输出结果
if count_arg == "status" then
    ngx.print(status_total," ",req_total)
 
elseif count_arg == "flow" then
    ngx.print(flow_total," ",req_total)
 
elseif count_arg == "reqT" then
    local reqt_avg = 0
    if req_total == 0 then
        reqt_avg = 0
    else
        reqt_avg = reqtime_total/req_total
    end
    ngx.print(reqt_avg," ",req_total)
 
elseif count_arg == "upT" then
    local upt_avg = 0
    if req_total == 0 then
            upt_avg = 0
    else
            upt_avg = upstream_total/req_total
    end
    ngx.print(upt_avg," ",req_total)
 
elseif count_arg == "statusUrl" then
    if status_total > tonumber(exceed_arg) then
        -- 排序table
        status_url_t_key = getKeysSortedByValue(status_url_t, function(a, b) return a > b end)
        local output_body = ""
        for i, uri in ipairs(status_url_t_key) do
            if output_body == "" then
                output_body = table.concat({uri," ",status_url_t[uri]})
            else    
                output_body = table.concat({output_body,"
",uri," ",status_url_t[uri]})
            end
            if i >= tonumber(output_arg) then
                ngx.print(output_body)
                ngx.exit(ngx.HTTP_OK)
            end                
        end
 
        ngx.print(output_body)
        ngx.exit(ngx.HTTP_OK)
    end
 
elseif count_arg == "upTUrl" then
    local max_output = 30
    local total_time = 0
    local total_count = 0
    local output_body = ""
    local i = 0
    for url in pairs(upstream_url_t) do
        i = i + 1
        total_time = upstream_url_t[url]
        total_count = upstream_url_count_t[url]
        avg_time = upstream_url_t[url] / upstream_url_count_t[url]
        if avg_time > tonumber(exceed_arg) then
            output_body = table.concat({url," ",avg_time," ",total_count,"
",output_body})
        end
 
        if i >= max_output then
            ngx.print(output_body)
            ngx.exit(ngx.HTTP_OK)
        end            
    end
    ngx.print(output_body)
    ngx.exit(ngx.HTTP_OK)
 
end