cadvisor采集docker容器的DiskUsage指标--源码分析

头像
a朋
     上海
    阅读 10 分钟
    1

    一. DiskUsage指标

    cadvisor采集的docker容器的DiskUsage指标，包含：

    • container_fs_inodes_free
    • container_fs_inodes_total
    • container_fs_limit_bytes
    • container_fs_usage_bytes
    // cadvisor/metrics/prometheus.go
func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetrics container.MetricSet, now clock.Clock, opts v2.RequestOptions) *PrometheusCollector {
    ...
    if includedMetrics.Has(container.DiskUsageMetrics) {
        c.containerMetrics = append(c.containerMetrics, []containerMetric{
            {
                name:        "container_fs_inodes_free",
                help:        "Number of available Inodes",
                valueType:   prometheus.GaugeValue,
                extraLabels: []string{"device"},
                getValues: func(s *info.ContainerStats) metricValues {
                    return fsValues(s.Filesystem, func(fs *info.FsStats) float64 {
                        return float64(fs.InodesFree)
                    }, s.Timestamp)
                },
            }, {
                name:        "container_fs_inodes_total",
                help:        "Number of Inodes",
                valueType:   prometheus.GaugeValue,
                extraLabels: []string{"device"},
                getValues: func(s *info.ContainerStats) metricValues {
                    return fsValues(s.Filesystem, func(fs *info.FsStats) float64 {
                        return float64(fs.Inodes)
                    }, s.Timestamp)
                },
            }, {
                name:        "container_fs_limit_bytes",
                help:        "Number of bytes that can be consumed by the container on this filesystem.",
                valueType:   prometheus.GaugeValue,
                extraLabels: []string{"device"},
                getValues: func(s *info.ContainerStats) metricValues {
                    return fsValues(s.Filesystem, func(fs *info.FsStats) float64 {
                        return float64(fs.Limit)
                    }, s.Timestamp)
                },
            }, {
                name:        "container_fs_usage_bytes",
                help:        "Number of bytes that are consumed by the container on this filesystem.",
                valueType:   prometheus.GaugeValue,
                extraLabels: []string{"device"},
                getValues: func(s *info.ContainerStats) metricValues {
                    return fsValues(s.Filesystem, func(fs *info.FsStats) float64 {
                        return float64(fs.Usage)
                    }, s.Timestamp)
                },
            },
        }...)
    }
    ...
}

    二. 采集的过程

    docker容器的Disk类型指标，由dockerFsHandler负责采集，它包含：

    • realFsHandler: 它包含：

      • rootfs: 镜像层中最上层，即可读可写层的disk耗费，通常存放在{rwLayId}/diff目录；
      • extraDir: 存放容器的日志文件目录的disk耗费；
    • thinPoolWatcher: 若storageDriver=DeviceMapper时，额外的Disk耗费；
    • zfsWatcher: 若storageDriver=Zfs时，额外的Disk耗费；

    上述三部分采集到的disk耗费的总和，即是某个docker容器的disk耗费。

    image.png

    1. dockerFsHandler的初始化

    初始化时，会先判断是否采集DiskUsage类型的指标：

    // cadvisor/container/docker/handler.go
func newDockerContainerHandler(
    client *docker.Client,
    name string,
    machineInfoFactory info.MachineInfoFactory,
    fsInfo fs.FsInfo,
    storageDriver storageDriver,
    storageDir string,
    cgroupSubsystems map[string]string,
    inHostNamespace bool,
    metadataEnvAllowList []string,
    dockerVersion []int,
    includedMetrics container.MetricSet,
    thinPoolName string,
    thinPoolWatcher *devicemapper.ThinPoolWatcher,
    zfsWatcher *zfs.ZfsWatcher,
) (container.ContainerHandler, error) {
    ...
    if includedMetrics.Has(container.DiskUsageMetrics) {        // 是否采集Disk类别的指标
        handler.fsHandler = &dockerFsHandler{
            fsHandler:       common.NewFsHandler(common.DefaultPeriod, rootfsStorageDir, otherStorageDir, fsInfo),
            thinPoolWatcher: thinPoolWatcher,
            zfsWatcher:      zfsWatcher,
            deviceID:        ctnr.GraphDriver.Data["DeviceId"],
            zfsFilesystem:   zfsFilesystem,
        }
    }
    ...
}

    其中：thinPoolWatcher和zfsWatcher的初始化：

    • docker较为常用的storageDriver为overlay2，故thinPoolWatcher和zfsWatcher为nil;
    // cadvisor/container/docker/factory.go
func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics container.MetricSet) error {
    ...
    if includedMetrics.Has(container.DiskUsageMetrics) {            // 采集Disk类别的指标
        if storageDriver(dockerInfo.Driver) == devicemapperStorageDriver {    // storageDriver=DeviceMapper
            thinPoolWatcher, err = startThinPoolWatcher(dockerInfo)
            if err != nil {
                klog.Errorf("devicemapper filesystem stats will not be reported: %v", err)
            }
            // Safe to ignore error - driver status should always be populated.
            status, _ := StatusFromDockerInfo(*dockerInfo)
            thinPoolName = status.DriverStatus[dockerutil.DriverStatusPoolName]
        }
        if storageDriver(dockerInfo.Driver) == zfsStorageDriver {     // storageDriver=Zfs
            zfsWatcher, err = startZfsWatcher(dockerInfo)
            if err != nil {
                klog.Errorf("zfs filesystem stats will not be reported: %v", err)
            }
        }
    }
    ...
}

    dockerFsHandler最重要的是字段是realFsHandler，其初始化为:

    common.NewFsHandler(common.DefaultPeriod, rootfsStorageDir, otherStorageDir, fsInfo)
    // cadvisor/container/common/fsHandler.go
func NewFsHandler(period time.Duration, rootfs, extraDir string, fsInfo fs.FsInfo) FsHandler {
   return &realFsHandler{
      lastUpdate: time.Time{},
      usage:      FsUsage{},
      period:     period,
      minPeriod:  period,
      rootfs:     rootfs,
      extraDir:   extraDir,
      fsInfo:     fsInfo,
      stopChan:   make(chan struct{}, 1),
   }
}

    2. dockerFsHandler.Usage()采集指标

    某个容器的disk耗费，由以下组成：

    • realFsHandler.Usage()；
    • 若storageDriver=DeviceMapper，增加thinPollWatcher.GetUsage();
    • 若storageDriver=Zfs，增加zfsWatcher.GetUsage();

    对于storageDriver=overlay2的docker容器，它的disk耗费，由realFsHandler.Usage()决定。

    // cadvisor/container/docker/handler.go
func (h *dockerFsHandler) Usage() common.FsUsage {
    usage := h.fsHandler.Usage()        // realFsHander

    // 若storageDriver=DeviceMapper
    // When devicemapper is the storage driver, the base usage of the container comes from the thin pool.
    if h.thinPoolWatcher != nil {
        thinPoolUsage, err := h.thinPoolWatcher.GetUsage(h.deviceID)
        if err != nil {
            ...
        } else {
            usage.BaseUsageBytes = thinPoolUsage
            usage.TotalUsageBytes += thinPoolUsage
        }
    }

    // 若storageDriver = Zfs
    if h.zfsWatcher != nil {
        zfsUsage, err := h.zfsWatcher.GetUsage(h.zfsFilesystem)
        if err != nil {
            klog.V(5).Infof("unable to get fs usage from zfs for filesystem %s: %v", h.zfsFilesystem, err)
        } else {
            usage.BaseUsageBytes = zfsUsage
            usage.TotalUsageBytes += zfsUsage
        }
    }
    return usage
}

    三. 如何确定docker的storageDriver类型

    cadvisor支持的storageDriver类型：

    // cadvisor/container/docker/factory.go
type storageDriver string

const (
   devicemapperStorageDriver storageDriver = "devicemapper"
   aufsStorageDriver         storageDriver = "aufs"
   overlayStorageDriver      storageDriver = "overlay"
   overlay2StorageDriver     storageDriver = "overlay2"
   zfsStorageDriver          storageDriver = "zfs"
   vfsStorageDriver          storageDriver = "vfs"
)

    具体到环境上，可以通过Docker Info命令查看StorageDriver:

    # docker info|grep Driver
 Storage Driver: overlay2
 Logging Driver: json-file
 Cgroup Driver: cgroupfs

    目前docker常用的storageDriver为Overlay2。

    四. realFsHandler.Usage()

    对于普通的docker容器，它的storageDriver=Overlay2，其disk耗费，由realFsHandler.Usage()决定。

    // cadvisor/container/common/fsHandler.go
func (fh *realFsHandler) Usage() FsUsage {
    fh.RLock()
    defer fh.RUnlock()
    return fh.usage        // 直接返回
}

    realFsHandler.usage由其中的update()函数负责采集：

    • inodeUsage是rootfs的inodes；
    • baseUsageBytes是rootfs的usageBytes；
    • totalUsagedBytes是rootfs + extraDir 的usage之和；
    // cadvisor/container/common/fsHandler.go
func (fh *realFsHandler) update() error {
    var (
        rootUsage, extraUsage fs.UsageInfo
        rootErr, extraErr     error
    )
    // TODO(vishh): Add support for external mounts.
    if fh.rootfs != "" {
        rootUsage, rootErr = fh.fsInfo.GetDirUsage(fh.rootfs)        // rootfs的Usage
    }
    if fh.extraDir != "" {
        extraUsage, extraErr = fh.fsInfo.GetDirUsage(fh.extraDir)    // extraDir的usage
    }

    // Wait to handle errors until after all operartions are run.
    // An error in one will not cause an early return, skipping others
    fh.Lock()
    defer fh.Unlock()
    fh.lastUpdate = time.Now()
    if fh.rootfs != "" && rootErr == nil {
        fh.usage.InodeUsage = rootUsage.Inodes        // inodeUsage=rootfs的usage
        fh.usage.BaseUsageBytes = rootUsage.Bytes     // baseUsage=rootfs的usage
        fh.usage.TotalUsageBytes = rootUsage.Bytes    // totalUsage=rootfs的usage
    }
    if fh.extraDir != "" && extraErr == nil {
        if fh.rootfs != "" {
            fh.usage.TotalUsageBytes += extraUsage.Bytes    // totalUsage += extraDir的usage
        } else {
            // rootfs is empty, totalUsageBytes use extra usage bytes
            fh.usage.TotalUsageBytes = extraUsage.Bytes
        }
    }
    // Combine errors into a single error to return
    if rootErr != nil || extraErr != nil {
        return fmt.Errorf("rootDiskErr: %v, extraDiskErr: %v", rootErr, extraErr)
    }
    return nil
}

    在上面的代码中:

    • rootfs和extraDir为目录;
    • rootfs是容器的可读可写层，即进程在容器filesystem内部的读写；
    • extraDir是容器的日志目录，通常存储在/var/lib/docker/containers/{containerId}；
    • GetDirUsage()函数遍历目录下的所有文件，然后计算目录下所有文件的总和：
    // cadvisor/fs/fs.go
func GetDirUsage(dir string) (UsageInfo, error) {
    var usage UsageInfo
    rootInfo, err := os.Stat(dir)
    rootStat, ok := rootInfo.Sys().(*syscall.Stat_t)
    rootDevID := rootStat.Dev
    // dedupedInode stores inodes that could be duplicates (nlink > 1)
    dedupedInodes := make(map[uint64]struct{})

    err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
        ...
        s, ok := info.Sys().(*syscall.Stat_t)
        ...
        if s.Nlink > 1 {
            if _, ok := dedupedInodes[s.Ino]; !ok {
                // Dedupe things that could be hardlinks
                dedupedInodes[s.Ino] = struct{}{}
                usage.Bytes += uint64(s.Blocks) * statBlockSize
                usage.Inodes++
            }
        } else {
            usage.Bytes += uint64(s.Blocks) * statBlockSize
            usage.Inodes++
        }
        return nil
    })
    return usage, err
}

    五. 总结

    从prometheus提供指标，到docker的realFsHandler采集到指标，其对应关系如下：

    • container_fs_inodes_free：由于FsStats.InodesFree的值没有被初始化，故对docker容器，该值始终为0；
    • container_fs_inodes_total: 最终被赋值为rootUsage.Inodes，即rootfs的inodes数量；

    • container_fs_usage_bytes: 最终被赋值为rootUsage.bytes + extraUsage.bytes，即rootfs和extra目录大小；

      • rootfs是容器的可读可写层目录；
      • extraDir是容器的日志目录；

    image.png

    参考：

    1.https://juejin.cn/post/6844903751111671815
    2.https://www.cnblogs.com/vinsent/p/15830271.html

    cadvisor
    a朋
    63 声望38 粉丝
    « 上一篇
    VictoriaMetrics源码：写入与索引
    下一篇 »
    cgroup blkio子系统对容器的读写限速

    引用和评论

    推荐阅读
    头像
    alertmanager源码：整体架构和流程分析

    a朋阅读 1.6k

    0 条评论
    得票最新