kubernetes ingress-nginx controller的指标获取方法

ingress-nginx controller组件暴露了/metrics接口，prometheus可以拉取它的指标。

本文重点关注其内部指标的获取方法。

一. 创建ingress

1. 创建deploy

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: nginx
  name: nginx-deploy
spec:
  replicas: 2    ##2副本
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - image: nginx:1.14
        imagePullPolicy: IfNotPresent
        name: nginx
      restartPolicy: Always

2. 创建service

apiVersion: v1
kind: Service
metadata:
  labels:
    app: nginx-svc
  name: nginx-svc
spec:
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 80
  selector:
    app: nginx
  sessionAffinity: None
  type: ClusterIP

3. 创建ingress

apiVersion: networking.k8s.io/v1 
kind: Ingress
metadata:
  annotations:
    kubernetes.io/ingress.class: "nginx"
  name: nginx-ingress
spec:
  rules:
  - host: example.com
    http:
      paths: 
      - backend:
          service:
            name: nginx-svc
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific

访问http://example.com:20004/，controller监听在hostNetwork的20004端口

4. 查看指标

二. 指标获取方法

重点关注以下指标：

• nginx_ingress_controller_request_size: ingress请求size
• nginx_ingress_controller_reseponse_size: ingress响应size

1. 指标定义

// ingress-nginx/internal/ingress/metric/collectors/socket.go
func NewSocketCollector(pod, namespace, class string, metricsPerHost bool) (*SocketCollector, error) {
    ...
    sc := &SocketCollector{
        ...
        responseLength: prometheus.NewHistogramVec(
            prometheus.HistogramOpts{
                Name:        "response_size",
                Help:        "The response length (including request line, header, and request body)",
                Namespace:   PrometheusNamespace,
                ConstLabels: constLabels,
            },
            requestTags,
        ),
        requestLength: prometheus.NewHistogramVec(
            prometheus.HistogramOpts{
                Name:        "request_size",
                Help:        "The request length (including request line, header, and request body)",
                Namespace:   PrometheusNamespace,
                Buckets:     prometheus.LinearBuckets(10, 10, 10), // 10 buckets, each 10 bytes wide.
                ConstLabels: constLabels,
            },
            requestTags,
        ),
        ...
    }
    ...
}

可以看到，这两个指标均是Histogram直方图类型。

2. 指标赋值

那这两个指标的值如何被赋值呢？

// ingress-nginx/internal/ingress/metric/collectors/socket.go
func (sc *SocketCollector) Start() {
    for {
        conn, err := sc.listener.Accept()
        if err != nil {
            continue
        }
        go handleMessages(conn, sc.handleMessage)
    }
}

func (sc *SocketCollector) handleMessage(msg []byte) {
    ...
    var statsBatch []socketData
    err := jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal(msg, &statsBatch)
    for _, stats := range statsBatch {
        ...
        if stats.RequestLength != -1 {
            requestLengthMetric, err := sc.requestLength.GetMetricWith(requestLabels)
            if err != nil {
                klog.Errorf("Error fetching request length metric: %v", err)
            } else {
                requestLengthMetric.Observe(stats.RequestLength)    // request_size
            }
        }
        if stats.ResponseLength != -1 {
            bytesSentMetric, err := sc.bytesSent.GetMetricWith(requestLabels)
            if err != nil {
                klog.Errorf("Error fetching bytes sent metric: %v", err)
            } else {
                bytesSentMetric.Observe(stats.ResponseLength)    // sent_bytes
            }

            responseSizeMetric, err := sc.responseLength.GetMetricWith(requestLabels)
            if err != nil {
                klog.Errorf("Error fetching bytes sent metric: %v", err)
            } else {
                responseSizeMetric.Observe(stats.ResponseLength)    // response_size
            }
        }
        ...
    }
}

从上面可以看出：

• 指标值从conn网络连接上读取而来，读到的数据是[]socketData类型；
• request_size的值来源于 socketData.RequestLength；
• respoonse_size的值来源于 socketData.ResponseLength;
• sent_bytes指标的值，跟response_size一样，均来源于 socketData.ReponseLength;

3. conn连接是哪里的数据

上面讲到，数据源是从conn网络连接上读到的[]socketData，那么conn是连接的谁呢？

// ingress-nginx/internal/ingress/metric/collectors/socket.go
func NewSocketCollector(pod, namespace, class string, metricsPerHost bool) (*SocketCollector, error) {
    socket := "/tmp/prometheus-nginx.socket"
    // unix sockets must be unlink()ed before being used
    _ = syscall.Unlink(socket)

    listener, err := net.Listen("unix", socket)
    ...
}

从上面可以看到：

• listener监听的unix socket文件：/tmp/prometheus-nginx.socket；
• conn也就是跟与socket的连接，数据也是通过socket连接读取到的；

那么谁会向这个socket写入数据呢？

4.ingress-nginx POD

在ingress-nginx的代码中没有找到写socket(/tmp/prometheus-nginx.socket)的地方，到ingress-nginx的POD内看看。

# kubectl exec -it -n ingress-nginx ingress-nginx-controller-j6mmt — sh
/etc/nginx $ grep -r "prometheus-nginx.socket" ./
./lua/monitor.lua:  assert(s:connect("unix:/tmp/prometheus-nginx.socket"))
./lua/test/monitor_test.lua:      assert.stub(tcp_mock.connect).was_called_with(tcp_mock, "unix:/tmp/prometheus-nginx.socket")

可以看到，nginx中有个monitor的lua插件，它会写socket: /tmp/prometheus-nginx.socket，监控指标应该就是从这里来的。

/etc/nginx $ vi ./lua/monitor.lua
…
local function send(payload)
  local s = assert(socket())
  assert(s:connect("unix:/tmp/prometheus-nginx.socket"))
  assert(s:send(payload))
  assert(s:close())
end

local function metrics()
  return {
    ....
    requestLength = tonumber(ngx.var.request_length) or -1,
    responseLength = tonumber(ngx.var.bytes_sent) or -1,
    ...
  }
...

可以看到：

• monitor.lua:

  • 将nginx的request_length和bytes_sent变量赋值给requestLength和responseLength；
  • 将数据组装成[]socketData发送到soket；
• ingress-nginx：从socket上读取[]socketData，然后给request_size、response_size指标赋值；

三. 总结

controller监听到Ingress对象被创建，将其配置reload到nignx上，然后使用lua脚本监控其读写流量，通过socket写指标写入controller，最后通过/metrics接口暴露出去：