如何使用prometheus监控nginx

基础运维 , Comments

[toc]

需求场景

使用prometheus监控nginx，图表展示使用grafana。

分析

nginx我们使用tengine(https://github.com/alibaba/tengine)

nginx prometheus模块使用nginx-module-vts(https://github.com/vozlt/nginx-module-vts)

exportor使用nginx-vts-exporter(https://github.com/hnlq715/nginx-vts-exporter)

(其实这个exportor也可以不用，因为默认的vts已经提供了prometheus的metrics地址，这个exportor只是为了生成这个地址)

nginx以及插件部署

下载文件

1
2
3
4
5
6
7
git clone https://github.com/alibaba/tengine
git clone https://github.com/vozlt/nginx-module-vts
yum -y install zlib* library*
yum -y install openssl openssl-devel
---------
[root@10-17-41-57 software]# ls
nginx-module-vts  tengine

构建tengine，带上vts

1
2
3
4
5
[root@10-17-41-57 software]# cd tengine
[root@10-17-41-57 tengine]# ./configure --add-module=/root/software/nginx-module-vts --prefix=/usr/local/nginx --user=nginx --group=nginx --with-stream --with-http_ssl_module --with-http_stub_status_module --with-http_realip_module --with-http_v2_module --with-http_ssl_module
[root@10-17-41-57 tengine]# make
[root@10-17-41-57 tengine]# make install
[root@10-17-41-57 tengine]# useradd www

配置nginx

nginx.conf中的http块中加入下面代码：

1
2
3
4
5
6
# 开启基础监控
vhost_traffic_status_zone;
# 开启状态码过滤
vhost_traffic_status_filter on;
# 开启此功能，在Nginx配置有多个server_name的情况下，会根据不同的server_name进行流量的统计，否则默认会把流量全部计算到第一个server_name上
vhost_traffic_status_filter_by_host on;

新增nginx-vts-status.conf文件：

1
2
3
4
5
6
7
8
9
server {        
    listen 3389;

    location /mt_status {
        vhost_traffic_status_display;
        vhost_traffic_status_display_format html;
    }

}

启动nginx：

1
2
[root@10-17-41-57 software]# /usr/local/nginx/sbin/nginx -t
[root@10-17-41-57 software]# /usr/local/nginx/sbin/nginx

查看原始nginx输出数据

1
2
3
4
5
6
7
8
9
10
11
12
13
14
[root@10-17-41-57 software]# curl https://localhost:3389/mt_status/format/prometheus

# HELP nginx_vts_start_time_seconds Nginx start time
# TYPE nginx_vts_start_time_seconds gauge
nginx_vts_start_time_seconds 1600401217.372
# HELP nginx_vts_main_connections Nginx connections
# TYPE nginx_vts_main_connections gauge
nginx_vts_main_connections{status="accepted"} 201
nginx_vts_main_connections{status="active"} 98
nginx_vts_main_connections{status="handled"} 201
nginx_vts_main_connections{status="reading"} 0
nginx_vts_main_connections{status="requests"} 10
nginx_vts_main_connections{status="waiting"} 97
nginx_vts_main_connections{status="writing"} 1

加一个测试的nginx配置文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
server {
    listen  80;
    server_name mtlab-nginx-test.meitu.com;

    proxy_ignore_client_abort on;
    # 开启详细状态码统计
    vhost_traffic_status_filter_by_set_key $status $server_name;
    # 开启uri统计
    vhost_traffic_status_filter_by_set_key $uri uris::$server_name;

    location / {
	return 200;
    }

}

配置好上述的参数，我们可以在输出看到一些更加详细的信息

png1

备注(nginx.conf文件)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
user  www www;

worker_processes auto;

error_log  /usr/local/nginx/logs/nginx_error.log  error;
pid    /usr/local/nginx/nginx.pid;

worker_rlimit_nofile 65535;

events
{
    use epoll;
    worker_connections 65535;
}

http {
    include       mime.types;
    default_type  application/octet-stream;
    vhost_traffic_status_zone;


    log_format api '$time_iso8601  $remote_addr  -  $host  "$request"  $status  -  $request_time  $http_x_real_ip  "$http_x_forwarded_for"  $content_length  $request_length  $sent_http_content_length  $body_bytes_sent  $http_cdn  "$http_referer"  "$http_user_agent"  $upstream_addr  $upstream_status  $upstream_response_time  $sent_http_request_id';

    log_format json '$time_iso8601  $remote_addr  -  $host  "$request"  $status  -  $request_time  $http_x_real_ip  "$http_x_forwarded_for"  $content_length  $request_length  $sent_http_content_length  $body_bytes_sent  $http_cdn  "$http_referer"  "$http_user_agent"  $upstream_addr  $upstream_status  $upstream_response_time  $sent_http_request_id';

    log_format mtlog '$time_iso8601  $remote_addr  $server_addr  $host  "$request"  $status  -  $request_time  $http_x_real_ip  "$http_x_forwarded_for"  $content_length  $request_length  $sent_http_content_length  $body_bytes_sent  $http_cdn  "$http_referer"  "$http_user_agent"  $upstream_addr  $upstream_status  $upstream_response_time  $sent_http_request_id $http_request_id';

    log_format toamtlog '$time_iso8601  $server_addr  $host  "$request"  $status  -  $request_time  $http_x_real_ip  "$http_x_forwarded_for"  $content_length  $request_length  $sent_http_content_length  $body_bytes_sent  $http_cdn  "$http_referer"  "$http_user_agent"  $upstream_addr  $upstream_status  $upstream_response_time  $sent_http_request_id $http_request_id';

    log_format mtlog_noarg '$time_iso8601  $remote_addr  $server_addr  $host  "$request_method $server_protocol"  $status  -  $request_time  $http_x_real_ip  "$http_x_forwarded_for"  $content_length  $request_length  $sent_http_content_length  $body_bytes_sent  $http_cdn  "$http_referer"  "$http_user_agent"  $upstream_addr  $upstream_status  $upstream_response_time  $sent_http_request_id  $http_request_id';

    log_format post '$time_iso8601  $remote_addr  $server_addr  $host  "$request"  $status  -  $request_time  $http_x_real_ip  "$http_x_forwarded_for"  $content_length  $request_length  $sent_http_content_length  $body_bytes_sent  $http_cdn  "$http_referer"  "$http_user_agent"  $upstream_addr  $upstream_status  $upstream_response_time  $sent_http_request_id  $request_body';

    server_tokens off;
#    charset  gb2312;

    server_names_hash_bucket_size 256;
    client_header_buffer_size 32k;
    large_client_header_buffers 4 32k;
    client_max_body_size 100m;
    client_body_buffer_size 10m;

    sendfile on;
    tcp_nopush     off;

    keepalive_timeout 600;
    keepalive_requests 10000;

    tcp_nodelay on;

    #proxy_connect_timeout 20s;
    #proxy_send_timeout 90s;
    #proxy_read_timeout 90s;
    proxy_connect_timeout 600s;
    proxy_send_timeout 600s;
    proxy_read_timeout 600s;
    proxy_buffer_size 64k;
    proxy_buffers 4 64k;
    proxy_busy_buffers_size 128k;
    proxy_temp_file_write_size 128k;
#    proxy_buffering off;
#    proxy_request_buffering off;

    gzip on;
    gzip_min_length  1k;
    gzip_buffers     4 16k;
    gzip_http_version 1.0;
    gzip_comp_level 4;
    gzip_types       text/plain application/x-javascript text/css application/xml text/xml application/json application/javascript;
    gzip_vary on;


    ssi on;
    ssi_silent_errors on;
    ssi_types text/shtml;
    ssi_last_modified on;

    underscores_in_headers  on;



    # The following is a sneaky way to do "set $the_real_ip $remote_addr"
    # Needed because using set is not allowed outside server blocks.

    # We can't use $proxy_add_x_forwarded_for because the realip module
    # replaces the remote_addr too soon
    map $http_x_forwarded_for $full_x_forwarded_for {

            default          "$http_x_forwarded_for, $remote_addr";

    }

    # ws.live.meitu.com
    map $http_upgrade $connection_upgrade {
        default upgrade;
        '' close;
    }

    include upstream/*.conf;
    include vhosts/*.conf;

    limit_req_status 403;
    limit_req_zone $remote_addr zone=mpstatplayvideo:100m rate=20r/s;
}

额外补充

如果你的nginx机器是多台，或者是动态的，那么这个是不能使用域名负载均衡来代理的，因为prometheus每次只能抓一条，即每次都只会获取一台机器的数据。

所以针对这个情况，两种方式：

1.如果你是容器化部署，那么他本身就有服务发现的机制，无所谓。

2.如果你是物理部署的，要嘛你写死prometheus的target配置，要嘛研究一下prometheus动态发现的原理。

猿的野生香蕉SRE

一个在运维道路上狂飙的老司机