如何使用prometheus监控nginx

[toc]

需求场景

使用prometheus监控nginx,图表展示使用grafana。

分析

nginx我们使用tengine(https://github.com/alibaba/tengine)

nginx prometheus模块使用nginx-module-vts(https://github.com/vozlt/nginx-module-vts)

exportor使用nginx-vts-exporter(https://github.com/hnlq715/nginx-vts-exporter)

(其实这个exportor也可以不用,因为默认的vts已经提供了prometheus的metrics地址,这个exportor只是为了生成这个地址)

nginx以及插件部署

下载文件

1
2
3
4
5
6
7
git clone https://github.com/alibaba/tengine
git clone https://github.com/vozlt/nginx-module-vts
yum -y install zlib* library*
yum -y install openssl openssl-devel
---------
[root@10-17-41-57 software]# ls
nginx-module-vts tengine

构建tengine,带上vts

1
2
3
4
5
[root@10-17-41-57 software]# cd tengine
[root@10-17-41-57 tengine]# ./configure --add-module=/root/software/nginx-module-vts --prefix=/usr/local/nginx --user=nginx --group=nginx --with-stream --with-http_ssl_module --with-http_stub_status_module --with-http_realip_module --with-http_v2_module --with-http_ssl_module
[root@10-17-41-57 tengine]# make
[root@10-17-41-57 tengine]# make install
[root@10-17-41-57 tengine]# useradd www

配置nginx

nginx.conf中的http块中加入下面代码:

1
2
3
4
5
6
# 开启基础监控
vhost_traffic_status_zone;
# 开启状态码过滤
vhost_traffic_status_filter on;
# 开启此功能,在Nginx配置有多个server_name的情况下,会根据不同的server_name进行流量的统计,否则默认会把流量全部计算到第一个server_name上
vhost_traffic_status_filter_by_host on;

新增nginx-vts-status.conf文件:

1
2
3
4
5
6
7
8
9
server {        
listen 3389;

location /mt_status {
vhost_traffic_status_display;
vhost_traffic_status_display_format html;
}

}

启动nginx:

1
2
[root@10-17-41-57 software]# /usr/local/nginx/sbin/nginx -t
[root@10-17-41-57 software]# /usr/local/nginx/sbin/nginx

查看原始nginx输出数据

1
2
3
4
5
6
7
8
9
10
11
12
13
14
[root@10-17-41-57 software]# curl https://localhost:3389/mt_status/format/prometheus

# HELP nginx_vts_start_time_seconds Nginx start time
# TYPE nginx_vts_start_time_seconds gauge
nginx_vts_start_time_seconds 1600401217.372
# HELP nginx_vts_main_connections Nginx connections
# TYPE nginx_vts_main_connections gauge
nginx_vts_main_connections{status="accepted"} 201
nginx_vts_main_connections{status="active"} 98
nginx_vts_main_connections{status="handled"} 201
nginx_vts_main_connections{status="reading"} 0
nginx_vts_main_connections{status="requests"} 10
nginx_vts_main_connections{status="waiting"} 97
nginx_vts_main_connections{status="writing"} 1

加一个测试的nginx配置文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
server {
listen 80;
server_name mtlab-nginx-test.meitu.com;

proxy_ignore_client_abort on;
# 开启详细状态码统计
vhost_traffic_status_filter_by_set_key $status $server_name;
# 开启uri统计
vhost_traffic_status_filter_by_set_key $uri uris::$server_name;

location / {
return 200;
}

}

配置好上述的参数,我们可以在输出看到一些更加详细的信息

png1

备注(nginx.conf文件)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
user  www www;

worker_processes auto;

error_log /usr/local/nginx/logs/nginx_error.log error;
pid /usr/local/nginx/nginx.pid;

worker_rlimit_nofile 65535;

events
{
use epoll;
worker_connections 65535;
}

http {
include mime.types;
default_type application/octet-stream;
vhost_traffic_status_zone;


log_format api '$time_iso8601 $remote_addr - $host "$request" $status - $request_time $http_x_real_ip "$http_x_forwarded_for" $content_length $request_length $sent_http_content_length $body_bytes_sent $http_cdn "$http_referer" "$http_user_agent" $upstream_addr $upstream_status $upstream_response_time $sent_http_request_id';

log_format json '$time_iso8601 $remote_addr - $host "$request" $status - $request_time $http_x_real_ip "$http_x_forwarded_for" $content_length $request_length $sent_http_content_length $body_bytes_sent $http_cdn "$http_referer" "$http_user_agent" $upstream_addr $upstream_status $upstream_response_time $sent_http_request_id';

log_format mtlog '$time_iso8601 $remote_addr $server_addr $host "$request" $status - $request_time $http_x_real_ip "$http_x_forwarded_for" $content_length $request_length $sent_http_content_length $body_bytes_sent $http_cdn "$http_referer" "$http_user_agent" $upstream_addr $upstream_status $upstream_response_time $sent_http_request_id $http_request_id';

log_format toamtlog '$time_iso8601 $server_addr $host "$request" $status - $request_time $http_x_real_ip "$http_x_forwarded_for" $content_length $request_length $sent_http_content_length $body_bytes_sent $http_cdn "$http_referer" "$http_user_agent" $upstream_addr $upstream_status $upstream_response_time $sent_http_request_id $http_request_id';

log_format mtlog_noarg '$time_iso8601 $remote_addr $server_addr $host "$request_method $server_protocol" $status - $request_time $http_x_real_ip "$http_x_forwarded_for" $content_length $request_length $sent_http_content_length $body_bytes_sent $http_cdn "$http_referer" "$http_user_agent" $upstream_addr $upstream_status $upstream_response_time $sent_http_request_id $http_request_id';

log_format post '$time_iso8601 $remote_addr $server_addr $host "$request" $status - $request_time $http_x_real_ip "$http_x_forwarded_for" $content_length $request_length $sent_http_content_length $body_bytes_sent $http_cdn "$http_referer" "$http_user_agent" $upstream_addr $upstream_status $upstream_response_time $sent_http_request_id $request_body';

server_tokens off;
# charset gb2312;

server_names_hash_bucket_size 256;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 100m;
client_body_buffer_size 10m;

sendfile on;
tcp_nopush off;

keepalive_timeout 600;
keepalive_requests 10000;

tcp_nodelay on;

#proxy_connect_timeout 20s;
#proxy_send_timeout 90s;
#proxy_read_timeout 90s;
proxy_connect_timeout 600s;
proxy_send_timeout 600s;
proxy_read_timeout 600s;
proxy_buffer_size 64k;
proxy_buffers 4 64k;
proxy_busy_buffers_size 128k;
proxy_temp_file_write_size 128k;
# proxy_buffering off;
# proxy_request_buffering off;

gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.0;
gzip_comp_level 4;
gzip_types text/plain application/x-javascript text/css application/xml text/xml application/json application/javascript;
gzip_vary on;


ssi on;
ssi_silent_errors on;
ssi_types text/shtml;
ssi_last_modified on;

underscores_in_headers on;



# The following is a sneaky way to do "set $the_real_ip $remote_addr"
# Needed because using set is not allowed outside server blocks.

# We can't use $proxy_add_x_forwarded_for because the realip module
# replaces the remote_addr too soon
map $http_x_forwarded_for $full_x_forwarded_for {

default "$http_x_forwarded_for, $remote_addr";

}

# ws.live.meitu.com
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}

include upstream/*.conf;
include vhosts/*.conf;

limit_req_status 403;
limit_req_zone $remote_addr zone=mpstatplayvideo:100m rate=20r/s;
}

额外补充

如果你的nginx机器是多台,或者是动态的,那么这个是不能使用域名负载均衡来代理的,因为prometheus每次只能抓一条,即每次都只会获取一台机器的数据。

所以针对这个情况,两种方式:

1.如果你是容器化部署,那么他本身就有服务发现的机制,无所谓。

2.如果你是物理部署的,要嘛你写死prometheus的target配置,要嘛研究一下prometheus动态发现的原理。