zap源码阅读(1)

头像
白沙云影
    阅读 14 分钟

    zap是 Uber 开发的一个高性能、强类型、分 level 的 go 语言日志库

    1.对象创建

    1.1 Logger 结构体

    type Logger struct {
    core zapcore.Core  // 核心接口

    development bool
    addCaller   bool 
    onFatal     zapcore.CheckWriteAction // default is WriteThenFatal

    name        string
    errorOutput zapcore.WriteSyncer

    addStack zapcore.LevelEnabler

    callerSkip int

    clock Clock
}

    Logger对象的创建有两种方式:

    1. 通过New 直接创建
    func New(core zapcore.Core, options ...Option) *Logger {
    if core == nil {
        return NewNop()
    }
    log := &Logger{
        core:        core,
        errorOutput: zapcore.Lock(os.Stderr),
        addStack:    zapcore.FatalLevel + 1,
        clock:       _systemClock,
    }
    return log.WithOptions(options...)
}

    函数WithOptions使用的是函数式选择模式, 和ants源码阅读（https://segmentfault.com/a/11... 中使用的方法一致。

    func (log *Logger) WithOptions(opts ...Option) *Logger {
    c := log.clone()
    for _, opt := range opts {
        opt.apply(c)
    }
    return c
}

    Option 结构

    type Option interface {
    apply(*Logger)
}

// optionFunc wraps a func so it satisfies the Option interface.
type optionFunc func(*Logger)

func (f optionFunc) apply(log *Logger) {
    f(log)
}

func WrapCore(f func(zapcore.Core) zapcore.Core) Option {
    return optionFunc(func(log *Logger) {
        log.core = f(log.core)
    })
}

func AddStacktrace(lvl zapcore.LevelEnabler) Option {
    return optionFunc(func(log *Logger) {
        log.addStack = lvl
    })
}
    1. 建造者模式 Build 创建
      建造者模式使用场景： 当一个类的构造函数参数个数超过4个，而且这些参数有些是可选的参数，考虑使用建造者模式。
    func (cfg Config) Build(opts ...Option) (*Logger, error) {
    enc, err := cfg.buildEncoder() // 创建Encoder对象, json or console
    if err != nil {
        return nil, err
    }
    sink, errSink, err := cfg.openSinks() // 创建正常输出和错误输出的文件路径
    if err != nil {
        return nil, err
    }

    if cfg.Level == (AtomicLevel{}) {
        return nil, fmt.Errorf("missing Level")
    }

    log := New(   
        zapcore.NewCore(enc, sink, cfg.Level),
        cfg.buildOptions(errSink)...,
    )  // 创建log对象
    if len(opts) > 0 {
        log = log.WithOptions(opts...) // 根据传入的配置重置log属性
    }
    return log, nil
}

    1.2 Config结构体解析

    type Config struct {

    Level AtomicLevel `json:"level" yaml:"level"` //Level是用来配置日志级别的，即日志的最低输出级别
    Development bool `json:"development" yaml:"development"` // 这个字段的含义是用来标记是否为开发者模式，在开发者模式下，日志输出的一些行为会和生产环境上不同。

    DisableCaller bool `json:"disableCaller" yaml:"disableCaller"` // 用来标记是否开启行号和文件名显示功能。

    DisableStacktrace bool `json:"disableStacktrace" yaml:"disableStacktrace"` //标记是否开启调用栈追踪能力，即在打印异常日志时，是否打印调用栈。

    Sampling *SamplingConfig `json:"sampling" yaml:"sampling"` // Sampling实现了日志的流控功能，或者叫采样配置，主要有两个配置参数，Initial和Thereafter，实现的效果是在1s的时间单位内，如果某个日志级别下同样内容的日志输出数量超过了Initial的数量，那么超过之后，每隔Thereafter的数量，才会再输出一次。是一个对日志输出的保护功能。

    Encoding string `json:"encoding" yaml:"encoding"` // json or console 

    EncoderConfig zapcore.EncoderConfig `json:"encoderConfig"  yaml:"encoderConfig"`

    OutputPaths []string `json:"outputPaths" yaml:"outputPaths"` // 日志输出路径

    ErrorOutputPaths []string `json:"errorOutputPaths"  // 错误日志输出路径
 yaml:"errorOutputPaths"`

    InitialFields map[string]interface{} `json:"initialFields" yaml:"initialFields"` // 初始化Fields
}

    EncoderConfig 结构

    type EncoderConfig struct {

    MessageKey    string `json:"messageKey" yaml:"messageKey"`
    LevelKey      string `json:"levelKey" yaml:"levelKey"`
    TimeKey       string `json:"timeKey" yaml:"timeKey"`
    NameKey       string `json:"nameKey" yaml:"nameKey"`
    CallerKey     string `json:"callerKey" yaml:"callerKey"`
    StacktraceKey string `json:"stacktraceKey" yaml:"stacktraceKey"`
    LineEnding    string `json:"lineEnding" yaml:"lineEnding"` // 配置行分隔符

    EncodeLevel    LevelEncoder    `json:"levelEncoder"  yaml:"levelEncoder"` 
    EncodeTime     TimeEncoder     `json:"timeEncoder" yaml:"timeEncoder"`
    EncodeDuration DurationEncoder `json:"durationEncoder" yaml:"durationEncoder"`
    EncodeCaller   CallerEncoder   `json:"callerEncoder" yaml:"callerEncoder"`
    EncodeName NameEncoder `json:"nameEncoder" yaml:"nameEncoder"`
}

    1.3 详解Build

    1.3.1 buildEncoder

    buildEncoder --> newEncoder --> zapcore.Encoder 接口
    newEncoder

    func newEncoder(name string, encoderConfig zapcore.EncoderConfig) (zapcore.Encoder, error) {
    if encoderConfig.TimeKey != "" && encoderConfig.EncodeTime == nil {
        return nil, fmt.Errorf("missing EncodeTime in EncoderConfig")
    }

    _encoderMutex.RLock()
    defer _encoderMutex.RUnlock() //使用读写锁
    if name == "" {
        return nil, errNoEncoderNameSpecified
    }
    constructor, ok := _encoderNameToConstructor[name] // 返回Encoder对象
    if !ok {
        return nil, fmt.Errorf("no encoder registered for name %q", name)
    }
    return constructor(encoderConfig)
}

    map中存储函数对象

        _encoderNameToConstructor = map[string]func(zapcore.EncoderConfig) (zapcore.Encoder, error){
        "console": func(encoderConfig zapcore.EncoderConfig) (zapcore.Encoder, error) {
            return zapcore.NewConsoleEncoder(encoderConfig), nil
        },
        "json": func(encoderConfig zapcore.EncoderConfig) (zapcore.Encoder, error) {
            return zapcore.NewJSONEncoder(encoderConfig), nil
        },
    }

    NewConsoleEncoder 创建的consoleEncoder结构体的底层结构是jsonEncoder, 只是 consoleEncoder 的spaced 为true, NewJSONEncoder的spaced 为false.

    jsonEncoder 结构体：

    type jsonEncoder struct {
    *EncoderConfig
    buf            *buffer.Buffer
    spaced         bool // include spaces after colons and commas
    openNamespaces int

    // for encoding generic values by reflection
    reflectBuf *buffer.Buffer
    reflectEnc *json.Encoder
}

func newJSONEncoder(cfg EncoderConfig, spaced bool) *jsonEncoder {
    return &jsonEncoder{
        EncoderConfig: &cfg,
        buf:           bufferpool.Get(),
        spaced:        spaced,
    }
}

    Buffer 结构体

    
const _size = 1024 // by default, create 1 KiB buffers

type Buffer struct {
    bs   []byte
    pool Pool
}

type Pool struct {
    p *sync.Pool
}

func NewPool() Pool {
    return Pool{p: &sync.Pool{
        New: func() interface{} {
            return &Buffer{bs: make([]byte, 0, _size)} //默认会创建1k的buffer
        },
    }}
}
    1.3.2 openSinks

    创建output, 和errorput 文件
    调用顺序: Open-->open--> newSink

    func (cfg Config) openSinks() (zapcore.WriteSyncer, zapcore.WriteSyncer, error) {
    sink, closeOut, err := Open(cfg.OutputPaths...)
    if err != nil {
        return nil, nil, err
    }
    errSink, _, err := Open(cfg.ErrorOutputPaths...)
    if err != nil {
        closeOut() // 下文中的close在这里被调用
        return nil, nil, err
    }
    return sink, errSink, nil
}

func Open(paths ...string) (zapcore.WriteSyncer, func(), error) {
    writers, close, err := open(paths)
    if err != nil {
        return nil, nil, err
    }

    writer := CombineWriteSyncers(writers...)
    return writer, close, nil
}

func open(paths []string) ([]zapcore.WriteSyncer, func(), error) {
    writers := make([]zapcore.WriteSyncer, 0, len(paths))
    closers := make([]io.Closer, 0, len(paths))
    close := func() { // 返回一个回调函数，使用巧妙
        for _, c := range closers {
            c.Close()
        }
    }

    var openErr error
    for _, path := range paths {
        sink, err := newSink(path) 
        if err != nil {
            openErr = multierr.Append(openErr, fmt.Errorf("couldn't open sink %q: %v", path, err))
            continue
        }
        writers = append(writers, sink) // 追加多个写入的路径
        closers = append(closers, sink) // 追加多个写入的路径用来关闭文件
    }
    if openErr != nil {
        close()
        return writers, nil, openErr
    }

    return writers, close, nil
}

func newSink(rawURL string) (Sink, error) {
    u, err := url.Parse(rawURL) // 因为路径还可以是http的url，正常的path 赋值到u.Path 上。
    if err != nil {
        return nil, fmt.Errorf("can't parse %q as a URL: %v", rawURL, err)
    }
    if u.Scheme == "" {
        u.Scheme = schemeFile
    }

    _sinkMutex.RLock()
    factory, ok := _sinkFactories[u.Scheme]
    _sinkMutex.RUnlock()
    if !ok {
        return nil, &errSinkNotFound{u.Scheme}
    }
    return factory(u)
}

    _sinkFactories 是个map结构，存储的是回调函数。

        _sinkFactories = map[string]func(*url.URL) (Sink, error){
        schemeFile: newFileSink,
    }
    
func newFileSink(u *url.URL) (Sink, error) {
    if u.User != nil {
        return nil, fmt.Errorf("user and password not allowed with file URLs: got %v", u)
    }
    if u.Fragment != "" {
        return nil, fmt.Errorf("fragments not allowed with file URLs: got %v", u)
    }
    if u.RawQuery != "" {
        return nil, fmt.Errorf("query parameters not allowed with file URLs: got %v", u)
    }
    // Error messages are better if we check hostname and port separately.
    if u.Port() != "" {
        return nil, fmt.Errorf("ports not allowed with file URLs: got %v", u)
    }
    if hn := u.Hostname(); hn != "" && hn != "localhost" {
        return nil, fmt.Errorf("file URLs must leave host empty or use localhost: got %v", u)
    }
    fmt.Println("u.Path: ", u.Path)
    switch u.Path {
    case "stdout":
        return nopCloserSink{os.Stdout}, nil // stdout
    case "stderr":
        return nopCloserSink{os.Stderr}, nil 
    }
    // 带有具体的输出文件名
    return os.OpenFile(u.Path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
}

    2.如何使用

    以 Info 为例:

    func (log *Logger) Info(msg string, fields ...Field) {
    if ce := log.check(InfoLevel, msg); ce != nil {
        ce.Write(fields...)
    }
}

    check 函数

    func (log *Logger) check(lvl zapcore.Level, msg string) *zapcore.CheckedEntry {
    // check must always be called directly by a method in the Logger interface
    // (e.g., Check, Info, Fatal).
    const callerSkipOffset = 2

    // Check the level first to reduce the cost of disabled log calls.
    // Since Panic and higher may exit, we skip the optimization for those levels.
    if lvl < zapcore.DPanicLevel && !log.core.Enabled(lvl) {
        return nil
    }

    // Create basic checked entry thru the core; this will be non-nil if the
    // log message will actually be written somewhere.
    ent := zapcore.Entry{
        LoggerName: log.name,
        Time:       log.clock.Now(),
        Level:      lvl,
        Message:    msg,
    }
    ce := log.core.Check(ent, nil) // 满足达到日志等级才新建CheckedEntry. 该checkedEntry 使用sync.Pool 创建。
    willWrite := ce != nil

    // Set up any required terminal behavior.
    switch ent.Level {
    case zapcore.PanicLevel:
        ce = ce.Should(ent, zapcore.WriteThenPanic)
    case zapcore.FatalLevel:
        onFatal := log.onFatal
        // Noop is the default value for CheckWriteAction, and it leads to
        // continued execution after a Fatal which is unexpected.
        if onFatal == zapcore.WriteThenNoop {
            onFatal = zapcore.WriteThenFatal
        }
        ce = ce.Should(ent, onFatal)
    case zapcore.DPanicLevel:
        if log.development {
            ce = ce.Should(ent, zapcore.WriteThenPanic)
        }
    }

    // Only do further annotation if we're going to write this message; checked
    // entries that exist only for terminal behavior don't benefit from
    // annotation.
    if !willWrite {
        return ce
    }
     // 其他省略
}

    Write 函数

    func (ce *CheckedEntry) Write(fields ...Field) {
    if ce == nil {
        return
    }

    if ce.dirty {
        if ce.ErrorOutput != nil {
            // Make a best effort to detect unsafe re-use of this CheckedEntry.
            // If the entry is dirty, log an internal error; because the
            // CheckedEntry is being used after it was returned to the pool,
            // the message may be an amalgamation from multiple call sites.
            fmt.Fprintf(ce.ErrorOutput, "%v Unsafe CheckedEntry re-use near Entry %+v.\n", ce.Time, ce.Entry)
            ce.ErrorOutput.Sync()
        }
        return
    }
    ce.dirty = true

    var err error
    for i := range ce.cores {
        err = multierr.Append(err, ce.cores[i].Write(ce.Entry, fields)) // 核心，写入格式化数据
    }
    if ce.ErrorOutput != nil {
        if err != nil {
            fmt.Fprintf(ce.ErrorOutput, "%v write error: %v\n", ce.Time, err)
            ce.ErrorOutput.Sync()
        }
    }

    should, msg := ce.should, ce.Message
    putCheckedEntry(ce)

    switch should {
    case WriteThenPanic:
        panic(msg)
    case WriteThenFatal:
        exit.Exit()
    case WriteThenGoexit:
        runtime.Goexit()
    }
}

    核心写数据 writeContext

    
func (c *ioCore) Write(ent Entry, fields []Field) error {
    buf, err := c.enc.EncodeEntry(ent, fields) // 把数据写入缓存
    if err != nil {
        return err
    }
    _, err = c.out.Write(buf.Bytes()) // 写入数据到文件
    buf.Free()
    if err != nil {
        return err
    }
    if ent.Level > ErrorLevel {
        // Since we may be crashing the program, sync the output. Ignore Sync
        // errors, pending a clean solution to issue #370.
        c.Sync()
    }
    return nil
}

func (c consoleEncoder) writeContext(line *buffer.Buffer, extra []Field) {
    context := c.jsonEncoder.Clone().(*jsonEncoder)
    defer func() {
        // putJSONEncoder assumes the buffer is still used, but we write out the buffer so
        // we can free it.
        context.buf.Free()
        putJSONEncoder(context)
    }()

    addFields(context, extra) // 把 Fields 写入到buffer中
    context.closeOpenNamespaces()
    if context.buf.Len() == 0 {
        return
    }

    c.addSeparatorIfNecessary(line)
    line.AppendByte('{')
    line.Write(context.buf.Bytes())
    line.AppendByte('}')
}

    3.性能优势

    1. 类型:Entry、CheckedEntry、Buffer等使用sync.Pool管理，极大地提升内存使用
    2. 缓存会提前分配1k, 用于数据写入。
    3. 并未使用encoding/json 把数据转换成json格式,而是自己拼装。
    4. 避免使用反射，定义Fields。

    4.编程tip

    1. 函数的书写

      type nopCore struct{}

// NewNopCore returns a no-op Core.
func NewNopCore() Core                                        { return nopCore{} }
// 函数正常应该是 func(n nopCore) Enabled(Level) bool, 但是把 n 去掉了.
func (nopCore) Enabled(Level) bool                            { return false }
func (n nopCore) With([]Field) Core  
// 函数的入参当不需要时可以用_ 表示
func (nopCore) Check(_ Entry, ce *CheckedEntry) *CheckedEntry { return ce }
func (nopCore) Write(Entry, []Field) error                    { return nil }
func (nopCore) Sync() error                                   { return nil }

    2. 回调函数的赋值
      定义：

      var _sinkFactories map[string]func(*url.URL) (Sink, error) // keyed by scheme
_sinkFactories = map[string]func(*url.URL) (Sink, error){
     schemeFile: newFileSink,
}

func newFileSink(u *url.URL) (Sink, error) {
 switch u.Path {
 case "stdout":
     return nopCloserSink{os.Stdout}, nil
 case "stderr":
     return nopCloserSink{os.Stderr}, nil
 }
 return os.OpenFile(u.Path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
}

      使用

    func newSink(rawURL string) (Sink, error) {
    u, err := url.Parse(rawURL)
    if err != nil {
        return nil, fmt.Errorf("can't parse %q as a URL: %v", rawURL, err)
    }
    if u.Scheme == "" {
        u.Scheme = schemeFile
    }

    _sinkMutex.RLock()
    factory, ok := _sinkFactories[u.Scheme]
    _sinkMutex.RUnlock()
    if !ok {
        return nil, &errSinkNotFound{u.Scheme}
    }
    return factory(u)
}
    1. 获取被调函数的行号，函数名等。
    func getCallerFrame(skip int) (frame runtime.Frame, ok bool) {
    const skipOffset = 2 // skip getCallerFrame and Callers

    pc := make([]uintptr, 1)
    numFrames := runtime.Callers(skip+skipOffset, pc)
    if numFrames < 1 {
        return
    }

    frame, _ = runtime.CallersFrames(pc).Next()
    return frame, frame.PC != 0
}

func (log *Logger) check(lvl zapcore.Level, msg string) *zapcore.CheckedEntry {
        frame, defined := getCallerFrame(log.callerSkip + callerSkipOffset)
        if !defined {
            fmt.Fprintf(log.errorOutput, "%v Logger.check error: failed to get caller\n", ent.Time.UTC())
            log.errorOutput.Sync()
        }

        ce.Entry.Caller = zapcore.EntryCaller{
            Defined:  defined,
            PC:       frame.PC,
            File:     frame.File,
            Line:     frame.Line,
            Function: frame.Function,
        }
}

    参考: 1. https://zhuanlan.zhihu.com/p/...

    日志源码学习go
    白沙云影
    1 声望3 粉丝

    一个专注于voip的频道

    « 上一篇
    ants源码阅读
    下一篇 »
    zap源码阅读(2)

    引用和评论

    推荐阅读
    头像
    opensips支持国产数据库(达梦，高斯，OB)

    白沙云影阅读 361

    头像
    日志-log4j2基于AsyncAppender的异步日志打印

    半夏之沫阅读 5.6k

    头像
    日志-log4j2基于AsyncLogger的异步日志打印

    半夏之沫赞 1阅读 5.2k

    头像
    Go 并发控制：singleflight 详解

    江湖十年赞 2阅读 697

    头像
    日志-log4j2日志框架源码学习

    半夏之沫赞 2阅读 2.4k

    头像
    Python 与 PostgreSQL 集成：深入 psycopg2 的应用与实践

    风流倜傥的伤痕赞 1阅读 469

    头像
    windows下golang 使用go-oci8连接orcale配置 goframe框架配置后可直接使用

    dd9527阅读 3.5k评论 1

    0 条评论
    得票最新