k8s hpa简介

HPA

HPA全称Horizontal Pod Autoscaling ，可以实现pod的水平自动化扩缩容，比如当POD中业务负载上升的时候，可以创建新的POD来保证业务系统稳定运行，当POD中业务负载下降的时候，可以销毁POD来提高资源利用率。

HPA目前支持四种类型的指标，分别是Resource、Object、External、Pods。其中在稳定版本autoscaling/v1中只支持对CPU指标的动态伸缩，在测试版本autoscaling/v2beta2中支持memory和自定义指标的动态伸缩，并以annotation的方式工作在autoscaling/v1版本中。

refer: https://zhuanlan.zhihu.com/p/...

metrics-server

k8s api-server的支持

    - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
    - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
    - --requestheader-allowed-names=front-proxy-client
    - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
    - --requestheader-extra-headers-prefix=X-Remote-Extra-
    - --requestheader-group-headers=X-Remote-Group
    - --requestheader-username-headers=X-Remote-User

notes:如果你未在 master 节点上运行 kube-proxy，则必须确保 kube-apiserver 启动参数中包含--enable-aggregator-routing=true

部署

metrics-server.yaml 参见下文，部署时注意nodeSelector是否匹配

      nodeSelector:
        kubernetes.io/os: linux
        kubernetes.io/arch: "amd64"

kubectl create -f metrics-server.yaml

metrics api

手动获取指标

kubectl get --raw /apis/metrics.k8s.io/v1beta1/pods

or

[root@master01 hpa]# curl -H "Authorization: Bearer $toke" -k https://172.31.133.26:6443/apis/|grep metrics -C 5
    {
      "name": "metrics.k8s.io",
      "versions": [
        {
          "groupVersion": "metrics.k8s.io/v1beta1",
          "version": "v1beta1"
        }
      ],
      "preferredVersion": {
        "groupVersion": "metrics.k8s.io/v1beta1",
        "version": "v1beta1"
      }
    }
  ]
}
[root@master01 hpa]# curl -H "Authorization: Bearer $toke" -k https://172.31.133.26:6443/apis/metrics.k8s.io/v1beta1/pods |grep php -C 7
    {
      "metadata": {
        "name": "php-apache-7bf64757b9-sh4rl",
        "namespace": "default",
        "selfLink": "/apis/metrics.k8s.io/v1beta1/namespaces/default/pods/php-apache-7bf64757b9-sh4rl",
      },
      "containers": [
        {
          "name": "php-apache",
          "usage": {
            "cpu": "31395n",
            "memory": "11988Ki"
          }
        }
      ]
    },

CPU HPA

创建应用

kubectl run php-apache --image=k8s.gcr.io/hpa-example --requests=cpu=200m --expose --port=80

创建hpa

kubectl autoscale deployment php-apache --cpu-percent=50 --min=1 --max=10

压测

kubectl run -i --tty load-generator --image=busybox /bin/sh

Hit enter for command prompt

while true; do wget -q -O- http://php-apache; done

自动弹性

[root@master01 msxu]# kubectl get hpa -w
NAME         REFERENCE               TARGETS   MINPODS   MAXPODS   REPLICAS   AGE
php-apache   Deployment/php-apache   0%/50%    1         10        1          52m
php-apache   Deployment/php-apache   493%/50%   1         10        1          52m
php-apache   Deployment/php-apache   493%/50%   1         10        4          52m
php-apache   Deployment/php-apache   493%/50%   1         10        8          52m
php-apache   Deployment/php-apache   493%/50%   1         10        10         53m
php-apache   Deployment/php-apache   54%/50%    1         10        10         53m
php-apache   Deployment/php-apache   49%/50%    1         10        10         54m
php-apache   Deployment/php-apache   44%/50%    1         10        10         56m
php-apache   Deployment/php-apache   0%/50%     1         10        10         57m
php-apache   Deployment/php-apache   0%/50%     1         10        10         61m
php-apache   Deployment/php-apache   0%/50%     1         10        9          61m
php-apache   Deployment/php-apache   0%/50%     1         10        9          62m
php-apache   Deployment/php-apache   0%/50%     1         10        1          62m

MEM HPA

创建HPA

apiVersion: autoscaling/v2beta1
kind: HorizontalPodAutoscaler
metadata:
  name: php-apache-hpa
  namespace: default
spec:
  scaleTargetRef:
    apiVersion: apps/v1beta1
    kind: Deployment
    name: php-apache
  minReplicas: 1
  maxReplicas: 3
  metrics:
  - type: Resource
    resource:
      name: memory
      targetAverageUtilization: 30

查看

[root@master01 hpa]# kubectl get hpa
NAME             REFERENCE               TARGETS   MINPODS   MAXPODS   REPLICAS   AGE
php-apache       Deployment/php-apache   0%/50%    1         10        1          5d17h
php-apache-hpa   Deployment/php-apache   11%/30%   1         3         1          5d17h
[root@master01 hpa]# kubectl describe  hpa php-apache-hpa 
Name:                                                     php-apache-hpa
Namespace:                                                default
Labels:                                                   <none>
Annotations:                                              <none>
CreationTimestamp:                                        Thu, 30 Apr 2020 17:57:40 +0800
Reference:                                                Deployment/php-apache
Metrics:                                                  ( current / target )
  resource memory on pods  (as a percentage of request):  11% (11546624) / 30%
Min replicas:                                             1
Max replicas:                                             3
Deployment pods:                                          1 current / 1 desired
Conditions:
  Type            Status  Reason              Message
  ----            ------  ------              -------
  AbleToScale     True    ReadyForNewScale    recommended size matches current size
  ScalingActive   True    ValidMetricFound    the HPA was able to successfully calculate a replica count from memory resource utilization (percentage of request)
  ScalingLimited  False   DesiredWithinRange  the desired count is within the acceptable range
Events:           <none>

yaml

metrics-server.yaml 0.3.6

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: system:aggregated-metrics-reader
  labels:
    rbac.authorization.k8s.io/aggregate-to-view: "true"
    rbac.authorization.k8s.io/aggregate-to-edit: "true"
    rbac.authorization.k8s.io/aggregate-to-admin: "true"
rules:
- apiGroups: ["metrics.k8s.io"]
  resources: ["pods", "nodes"]
  verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: metrics-server:system:auth-delegator
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:auth-delegator
subjects:
- kind: ServiceAccount
  name: metrics-server
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: metrics-server-auth-reader
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: extension-apiserver-authentication-reader
subjects:
- kind: ServiceAccount
  name: metrics-server
  namespace: kube-system
---
apiVersion: apiregistration.k8s.io/v1beta1
kind: APIService
metadata:
  name: v1beta1.metrics.k8s.io
spec:
  service:
    name: metrics-server
    namespace: kube-system
  group: metrics.k8s.io
  version: v1beta1
  insecureSkipTLSVerify: true
  groupPriorityMinimum: 100
  versionPriority: 100
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: metrics-server
  namespace: kube-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: metrics-server
  namespace: kube-system
  labels:
    k8s-app: metrics-server
spec:
  selector:
    matchLabels:
      k8s-app: metrics-server
  template:
    metadata:
      name: metrics-server
      labels:
        k8s-app: metrics-server
    spec:
      serviceAccountName: metrics-server
      volumes:
      # mount in tmp so we can safely use from-scratch images and/or read-only containers
      - name: tmp-dir
        emptyDir: {}
      containers:
      - name: metrics-server
        image: k8s.gcr.io/metrics-server-amd64:v0.3.1
        imagePullPolicy: IfNotPresent
        args:
          - --cert-dir=/tmp
          - --secure-port=4443
          - --kubelet-preferred-address-types=InternalIP
          - --kubelet-insecure-tls
        ports:
        - name: main-port
          containerPort: 4443
          protocol: TCP
        securityContext:
          readOnlyRootFilesystem: true
          runAsNonRoot: true
          runAsUser: 1000
        volumeMounts:
        - name: tmp-dir
          mountPath: /tmp
      nodeSelector:
        kubernetes.io/os: linux
        kubernetes.io/arch: "amd64"
---
apiVersion: v1
kind: Service
metadata:
  name: metrics-server
  namespace: kube-system
  labels:
    kubernetes.io/name: "Metrics-server"
    kubernetes.io/cluster-service: "true"
spec:
  selector:
    k8s-app: metrics-server
  ports:
  - port: 443
    protocol: TCP
    targetPort: main-port
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: system:metrics-server
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - nodes
  - nodes/stats
  - namespaces
  - configmaps
  verbs:
  - get
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: system:metrics-server
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:metrics-server
subjects:
- kind: ServiceAccount
  name: metrics-server
  namespace: kube-system