容器运行时docker和containerd兼容问题

背景

k8s 1.20 默认采用containerd作为容器运行时

• https://kubernetes.io/zh/blog...

containerd 和 docker兼容问题

• 如果项目中需要用到底层 docker-api 则需要考虑兼容问题

k8s-mon项目 简介

• 是什么：滴滴夜莺Kubernetes monitor
• 项目地址 https://github.com/n9e/k8s-mon

k8s-mon为什么要兼容containerd

因为在k8s-mon中采集容器基础资源指标需要和夜莺的服务树打通
需要获取容器服务树标签N9E_NID

默认版本是采用docker inspect接口获取的

package collect

import (
    "context"
    "github.com/docker/docker/api/types"
    "github.com/docker/docker/client"
)

func getLabelMapByDockerSdk(nidLabelName string) (map[string]map[string]string, error) {
    ctx := context.Background()
    cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
    if err != nil {
        return nil, err
    }
    defer cli.Close()
    containers, err := cli.ContainerList(ctx, types.ContainerListOptions{})
    if err != nil {
        return nil, err
    }

    insM := make(map[string]map[string]string)
    whiteKeyM := map[string]struct{}{
        nidLabelName: {},
    }
    for _, container := range containers {
        podName := container.Labels["io.kubernetes.pod.name"]

        if podName == "" {
            continue
        }

        lastM, loaded := insM[podName]
        if !loaded {
            lastM = make(map[string]string)
        }

        for k, v := range container.Labels {

            if _, found := whiteKeyM[k]; !found {
                continue
            }
            lastM[k] = v
        }

        insM[podName] = lastM

    }
    return insM, nil
}

需要兼容下containerd的接口获取

package collect

import (
    "context"
    "errors"
    "github.com/containerd/containerd"
)

func getLabelMapByContainerdSdk(nidLabelName string) (map[string]map[string]string, error) {
    client, err := containerd.New("/run/containerd/containerd.sock", containerd.WithDefaultNamespace("k8s.io"))

    if err != nil {
        return nil, err
    }

    defer client.Close()

    context := context.Background()
    cers, err := client.Containers(context)
    if err != nil {
        return nil, err
    }
    if len(cers) == 0 {
        return nil, errors.New("got zero containers on this node")
    }

    insM := make(map[string]map[string]string)
    whiteKeyM := map[string]struct{}{
        nidLabelName: {},
    }

    for _, c := range cers {
        labels, err := c.Labels(context)
        if err != nil {
            continue
        }

        podName := labels["io.kubernetes.pod.name"]
        if podName == "" {
            continue
        }

        lastM, loaded := insM[podName]
        if !loaded {
            lastM = make(map[string]string)
        }

        for k, v := range labels {

            if _, found := whiteKeyM[k]; !found {
                continue
            }
            lastM[k] = v
        }

        insM[podName] = lastM

    }
    return insM, nil

}

k8s-mon获取容器tag时需要适配，默认采用docker-api，失败再尝试containerd-api

docker 镜像转成containerd的

ctr 命令执行时需要带上 --namespace k8s.io

docker build

docker build -t k8s-mon:v1 . && docker save  k8s-mon > a.tar

ctr 导入docker 镜像

ctr --namespace k8s.io images import a.tar

ctr 给镜像打tag = docker tag 4d8690b7763d registry.cn-beijing.aliyuncs.com/n9e/k8s-mon:v2.1.0

ctr --namespace k8s.io images tag docker.io/library/k8s-mon:v1 k8s-mon:v1

ctr get镜像 = docker images

[root@k8s-node01 ~]# ctr --namespace k8s.io images ls  
REF                                                                                                                                           TYPE                                                      DIGEST                                                                  SIZE      PLATFORMS                                                      LABELS                          
docker.io/eipwork/kuboard:latest                                                                                                              application/vnd.docker.distribution.manifest.v2+json      sha256:7e3d463e856d4d46f88b3241946554b144e3c285953e5055f6a6d6fa2f6360ad 69.0 MiB  linux/amd64                                                    io.cri-containerd.image=managed 
docker.io/eipwork/kuboard@sha256:7e3d463e856d4d46f88b3241946554b144e3c285953e5055f6a6d6fa2f6360ad                                             application/vnd.docker.distribution.manifest.v2+json      sha256:7e3d463e856d4d46f88b3241946554b144e3c285953e5055f6a6d6fa2f6360ad 69.0 MiB  linux/amd64                                                    io.cri-containerd.image=managed 
docker.io/eipwork/metrics-server:v0.3.7                                                                                                       application/vnd.docker.distribution.manifest.list.v2+json sha256:2a8e43d89e434b6235858444bb0701c44bd03475d2937fdf6831a7bb4ee14240 20.1 MiB  linux/amd64,linux/arm64                                        io.cri-containerd.image=managed 
docker.io/eipwork/metrics-server@sha256:2a8e43d89e434b6235858444bb0701c44bd03475d2937fdf6831a7bb4ee14240                                      application/vnd.docker.distribution.manifest.list.v2+json sha256:2a8e43d89e434b6235858444bb0701c44bd03475d2937fdf6831a7bb4ee14240 20.1 MiB  linux/amd64,linux/arm64                                        io.cri-containerd.image=managed 
docker.io/library/k8s-mon:v1                                

ctr 获取容器 = docker ps

[root@k8s-node01 ~]# ctr --namespace k8s.io c ls  
CONTAINER                                                           IMAGE                                                                                                                              RUNTIME                  
1d79feb8854e66f0c7c21739f7ee048edc1e278ce76d66c52981ea878d6eb231    registry.aliyuncs.com/k8sxio/pause@sha256:927d98197ec1141a368550822d18fa1c60bdae27b78b0c004f705f548c07814f                         io.containerd.runc.v2    
22baad7dc66daf480b8498fe8c31e546bc87f27b3b45807f6c8f942fccc9b686    sha256:919a16510f41bb12c86508529f915b123a5ff589aebb0a4355614a829aa47d62                                                            io.containerd.runc.v2    
2d0eefeec5fa5bf6b4bdc41933eb251b9550310b71f1cc300a2d05710cbcd5e7    quay.io/coreos/kube-state-metrics:v1.9.7                                                                                           io.containerd.runc.v2    
492a71bfeba379a6310095c888c21da99f1badb8253a9d57bb4536d4a2e31b82    swr.cn-east-2.myhuaweicloud.com/kuboard-dependency/node@sha256:304dd23bcda5216026f1601cb61395792249f1c58c98771198f6e517b0f5c96b    io.containerd.runc.v2    
4ebc6262d5217ea9b4f639051cb978398d2bcf38cd68b7bb83a20f270057bc2b    sha25

本地build 传输到远端机器

docker build -t k8s-mon:v1 . && docker save  k8s-mon > a.tar && nc 172.20.70.215 444  < a.tar
# 远端机器 
nc -l 444 >   a.tar && docker load < a.tar

push到阿里云

docker push registry.cn-beijing.aliyuncs.com/n9e/k8s-mon:v2.0.7

登录到阿里云

docker login --username=燕小乙1875 registry.cn-beijing.aliyuncs.com

删除僵尸容器

docker rmi $(docker images | grep "none" | awk '{print $3}')