聊聊golang的zap的encoder

序

本文主要研究一下golang的zap的encoder

encoder

zap@v1.16.0/zapcore/encoder.go

type Encoder interface {
    ObjectEncoder

    // Clone copies the encoder, ensuring that adding fields to the copy doesn't
    // affect the original.
    Clone() Encoder

    // EncodeEntry encodes an entry and fields, along with any accumulated
    // context, into a byte buffer and returns it. Any fields that are empty,
    // including fields on the `Entry` type, should be omitted.
    EncodeEntry(Entry, []Field) (*buffer.Buffer, error)
}

Encoder接口内嵌了ObjectEncoder，定义了Clone、EncodeEntry方法

ObjectEncoder

zap@v1.16.0/zapcore/encoder.go

type ObjectEncoder interface {
    // Logging-specific marshalers.
    AddArray(key string, marshaler ArrayMarshaler) error
    AddObject(key string, marshaler ObjectMarshaler) error

    // Built-in types.
    AddBinary(key string, value []byte)     // for arbitrary bytes
    AddByteString(key string, value []byte) // for UTF-8 encoded bytes
    AddBool(key string, value bool)
    AddComplex128(key string, value complex128)
    AddComplex64(key string, value complex64)
    AddDuration(key string, value time.Duration)
    AddFloat64(key string, value float64)
    AddFloat32(key string, value float32)
    AddInt(key string, value int)
    AddInt64(key string, value int64)
    AddInt32(key string, value int32)
    AddInt16(key string, value int16)
    AddInt8(key string, value int8)
    AddString(key, value string)
    AddTime(key string, value time.Time)
    AddUint(key string, value uint)
    AddUint64(key string, value uint64)
    AddUint32(key string, value uint32)
    AddUint16(key string, value uint16)
    AddUint8(key string, value uint8)
    AddUintptr(key string, value uintptr)

    // AddReflected uses reflection to serialize arbitrary objects, so it can be
    // slow and allocation-heavy.
    AddReflected(key string, value interface{}) error
    // OpenNamespace opens an isolated namespace where all subsequent fields will
    // be added. Applications can use namespaces to prevent key collisions when
    // injecting loggers into sub-components or third-party libraries.
    OpenNamespace(key string)
}

ObjectEncoder接口定义了各种类型的一系列Add方法

MapObjectEncoder

zap@v1.16.0/zapcore/memory_encoder.go

type MapObjectEncoder struct {
    // Fields contains the entire encoded log context.
    Fields map[string]interface{}
    // cur is a pointer to the namespace we're currently writing to.
    cur map[string]interface{}
}

MapObjectEncoder实现了ObjectEncoder接口，内部使用map[string]interface{}来存放数据

jsonEncoder

zap@v1.16.0/zapcore/json_encoder.go

type jsonEncoder struct {
    *EncoderConfig
    buf            *buffer.Buffer
    spaced         bool // include spaces after colons and commas
    openNamespaces int

    // for encoding generic values by reflection
    reflectBuf *buffer.Buffer
    reflectEnc *json.Encoder
}

jsonEncoder内嵌了EncoderConfig，定义了buf、spaced、openNamespaces、reflectBuf、reflectEnc属性

Clone

zap@v1.16.0/zapcore/json_encoder.go

func (enc *jsonEncoder) Clone() Encoder {
    clone := enc.clone()
    clone.buf.Write(enc.buf.Bytes())
    return clone
}

func (enc *jsonEncoder) clone() *jsonEncoder {
    clone := getJSONEncoder()
    clone.EncoderConfig = enc.EncoderConfig
    clone.spaced = enc.spaced
    clone.openNamespaces = enc.openNamespaces
    clone.buf = bufferpool.Get()
    return clone
}

Clone方法执行内部的clone，然后将原enc的buf拷贝到clone的encoder

EncodeEntry

zap@v1.16.0/zapcore/json_encoder.go

func (enc *jsonEncoder) EncodeEntry(ent Entry, fields []Field) (*buffer.Buffer, error) {
    final := enc.clone()
    final.buf.AppendByte('{')

    if final.LevelKey != "" {
        final.addKey(final.LevelKey)
        cur := final.buf.Len()
        final.EncodeLevel(ent.Level, final)
        if cur == final.buf.Len() {
            // User-supplied EncodeLevel was a no-op. Fall back to strings to keep
            // output JSON valid.
            final.AppendString(ent.Level.String())
        }
    }
    if final.TimeKey != "" {
        final.AddTime(final.TimeKey, ent.Time)
    }
    if ent.LoggerName != "" && final.NameKey != "" {
        final.addKey(final.NameKey)
        cur := final.buf.Len()
        nameEncoder := final.EncodeName

        // if no name encoder provided, fall back to FullNameEncoder for backwards
        // compatibility
        if nameEncoder == nil {
            nameEncoder = FullNameEncoder
        }

        nameEncoder(ent.LoggerName, final)
        if cur == final.buf.Len() {
            // User-supplied EncodeName was a no-op. Fall back to strings to
            // keep output JSON valid.
            final.AppendString(ent.LoggerName)
        }
    }
    if ent.Caller.Defined {
        if final.CallerKey != "" {
            final.addKey(final.CallerKey)
            cur := final.buf.Len()
            final.EncodeCaller(ent.Caller, final)
            if cur == final.buf.Len() {
                // User-supplied EncodeCaller was a no-op. Fall back to strings to
                // keep output JSON valid.
                final.AppendString(ent.Caller.String())
            }
        }
        if final.FunctionKey != "" {
            final.addKey(final.FunctionKey)
            final.AppendString(ent.Caller.Function)
        }
    }
    if final.MessageKey != "" {
        final.addKey(enc.MessageKey)
        final.AppendString(ent.Message)
    }
    if enc.buf.Len() > 0 {
        final.addElementSeparator()
        final.buf.Write(enc.buf.Bytes())
    }
    addFields(final, fields)
    final.closeOpenNamespaces()
    if ent.Stack != "" && final.StacktraceKey != "" {
        final.AddString(final.StacktraceKey, ent.Stack)
    }
    final.buf.AppendByte('}')
    if final.LineEnding != "" {
        final.buf.AppendString(final.LineEnding)
    } else {
        final.buf.AppendString(DefaultLineEnding)
    }

    ret := final.buf
    putJSONEncoder(final)
    return ret, nil
}

EncodeEntry方法先执行clone拷贝一份encoder，然后在拷贝出来的encoder上进行各种addKey及AppendString操作，最后拼接完通过putJSONEncoder将该encoder归还到_jsonPool

实例

func encoderDemo() {
    zap.RegisterEncoder("my-encoder", constructMyEncoder)
    // buf := &bytes.Buffer{}
    cfg := zap.NewDevelopmentConfig()
    cfg.Encoding = "my-encoder"
    logger, err := cfg.Build()
    defer logger.Sync()
    if err != nil {
        panic(err)
    }
    logger.Info("hello")
    logger.Info("failed to fetch URL",
        // Structured context as strongly typed Field values.
        zap.String("url", "https://example.com"),
        zap.Int("attempt", 3),
        zap.Duration("backoff", time.Second),
    )
}

type MyEncoder struct {
    *zapcore.MapObjectEncoder
}

var (
    _pool = buffer.NewPool()
    // Get retrieves a buffer from the pool, creating one if necessary.
    Get = _pool.Get
)

func constructMyEncoder(config zapcore.EncoderConfig) (zapcore.Encoder, error) {
    return MyEncoder{
        MapObjectEncoder: zapcore.NewMapObjectEncoder(),
    }, nil
}

func (enc MyEncoder) Clone() zapcore.Encoder {
    return MyEncoder{
        MapObjectEncoder: zapcore.NewMapObjectEncoder(),
    }
}

func (enc MyEncoder) EncodeEntry(entry zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) {
    myEnc := enc.Clone().(MyEncoder)
    buf := _pool.Get()

    buf.AppendString(entry.Message)
    buf.AppendString(" ")

    for _, field := range fields {
        field.AddTo(myEnc)
        value := myEnc.MapObjectEncoder.Fields[field.Key]
        buf.AppendString(field.Key)
        buf.AppendString("=")
        if value == "" {
            buf.AppendString(" ''")
        } else {
            buf.AppendString(fmt.Sprintf("%v ", value))
        }
    }

    buf.AppendByte('\n')

    if entry.Stack != "" {
        buf.AppendString(entry.Stack)
        buf.AppendByte('\n')
    }
    return buf, nil
}

这里定义了MyEncoder实现了Encoder接口，然后通过zap.RegisterEncoder进行注册，之后创建logger的时候设置Config的Encoding就可以根据指定的name找到对应的方法进行创建

小结

Encoder接口内嵌了ObjectEncoder，定义了Clone、EncodeEntry方法；ObjectEncoder接口定义了各种类型的一系列Add方法；MapObjectEncoder实现了ObjectEncoder接口，内部使用map[string]interface{}来存放数据。

doc

• zap