beyla源码: 从ebpf到metrics

beyla是Grafana开发一款基于ebpf的应用检测工具，它可以在不侵入应用程序代码的基础上，通过监听系统调用和用户函数，将其解析为RED指标，从而为应用程序提供可观测性。

本文以golang的nethttp程序为例，介绍beyla中数据采集、转换，最终到prometheus metrics的过程。

一. 整体流程

整体流程如下图所示：

• 首先，ebpf程序产生Events，由用户程序*.go读到；
• 然后，由ringbufReader读取Event信息，将其转换为ringbuf.Record；
• 再后，由binaryReader读取ringbuf.Record，将其转换为HTTPRequestTrace；
• 再后，由HTTPRequestTraceToSpan函数，将HTTPRequestTrace转换为request.Span；
• 最后，由metricsReporter读取requst.Span，将其转换为prometheus的metrics；

二. ebpf程序产生Events

1.内核态ebpf程序

ebpf程序中events的定义：

// beyla/bpf/ringbuf.h

struct {
    __uint(type, BPF_MAP_TYPE_RINGBUF);
    __uint(max_entries, 1 << 24);
} events SEC(".maps");

ebpf程序处理并保存event的代码:

// beyla/bpf/go_nethttp.c

SEC("uprobe/roundTrip_return")
int uprobe_roundTripReturn(struct pt_regs *ctx) {
    ...
    // 构造http_request_trace对象
    http_request_trace *trace = bpf_ringbuf_reserve(&events, sizeof(http_request_trace), 0);
    task_pid(&trace->pid);
    trace->type = EVENT_HTTP_CLIENT;
    trace->start_monotime_ns = invocation->start_monotime_ns;
    trace->go_start_monotime_ns = invocation->start_monotime_ns;
    trace->end_monotime_ns = bpf_ktime_get_ns();

    // Read arguments from the original set of registers
    // Get request/response struct
    ...
    // Get method from Request.Method
    ...
    // Get the host information of the remote
    ...
    // Get path from Request.URL
    ...
    // submit the completed trace via ringbuffer
    bpf_ringbuf_submit(trace, get_flags());            // submit到ringbuf
    return 0;
}

events中包含的http_request_trace的定义：

// beyla/bpf/http_trace.h

// Trace of an HTTP call invocation. It is instantiated by the return uprobe and forwarded to the
// user space through the events ringbuffer.
typedef struct http_request_trace_t {
    u8  type;                           // Must be first
    u64 go_start_monotime_ns;
    u64 start_monotime_ns;
    u64 end_monotime_ns;
    u8  method[METHOD_MAX_LEN];
    u8  path[PATH_MAX_LEN];
    u16 status;
    u8  remote_addr[REMOTE_ADDR_MAX_LEN];
    u64 remote_addr_len;
    u8  host[HOST_LEN];
    u64 host_len;
    u32 host_port;
    s64 content_length;
    tp_info_t tp;

    pid_info pid;
} __attribute__((packed)) http_request_trace;

2.用户态程序

用户态程序写在nethttp.go中，其中添加了go generate注解：

//go:generate $BPF2GO -cc $BPF_CLANG -cflags $BPF_CFLAGS -target amd64,arm64 bpf ../../../../bpf/go_nethttp.c -- -I../../../../bpf/headers
//go:generate $BPF2GO -cc $BPF_CLANG -cflags $BPF_CFLAGS -target amd64,arm64 bpf_debug ../../../../bpf/go_nethttp.c -- -I../../../../bpf/headers -DBPF_DEBUG

执行go generate后，将生成：

• bpf_bpfel_x86.go
• bpf_bpfel_x86.o

在bpf_bpfel_x86.go中：

• bpfObjects表示加载到内核的所有对象，其中包含bpfMaps；
• bpfMaps中包含Events数据；

// bpfObjects contains all objects after they have been loaded into the kernel.
//
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
type bpfObjects struct {
   bpfPrograms
   bpfMaps
}
// bpfMaps contains all maps after they have been loaded into the kernel.
//
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
type bpfMaps struct {
   Events                    *ebpf.Map `ebpf:"events”`
   ...
}

三. ringbufReader将Event转换为ringbuf.Record

整体的代码入口：

// nethttp.go

func (p *Tracer) Run(ctx context.Context, eventsChan chan<- []request.Span, service svc.ID) {
   logger := slog.With("component", "nethttp.Tracer")
   ebpfcommon.ForwardRingbuf[ebpfcommon.HTTPRequestTrace](
      service,
      p.Cfg, logger, p.bpfObjects.Events,
      ebpfcommon.ReadHTTPRequestTraceAsSpan,
      p.Metrics,
      append(p.closers, &p.bpfObjects)...,
   )(ctx, eventsChan)
}

ringbufReader负责：

• 从ringbuf中读出记录，保存为ringBuf.Record对象：

// beyla/pkg/internal/ebpf/common/ringbuf.go

func (rbf *ringBufForwarder[T]) readAndForward(ctx context.Context, spansChan chan<- []request.Span) {
    ...
    // BPF will send each measured trace via Ring Buffer, so we listen for them from the
    // user space.
    eventsReader, err := readerFactory(rbf.ringbuffer)
    ...
    record, err := eventsReader.Read()
    for {
        ...
        rbf.processAndForward(record, spansChan)


        // read another event before the next loop iteration
        record, err = eventsReader.Read()
    }
}

具体的ringbuf读取细节，由cilium的lib负责：

// cilium/ebpf/ringbuf/reader.go
// Read a record from an event ring.
//
// buf must be at least ringbufHeaderSize bytes long.
func readRecord(rd *ringbufEventRing, rec *Record, buf []byte) error {
    ...
}

四. binaryReader将ringbuf.Record转换为HTTPRequestTrace

代码入口：

• 首先，将ringbuf.Record转换为HTTPRequestTrace；

• 然后，将HTTPRequestTrace转换为request.Span；

  // pkg/internal/ebpf/common/common.go
  func ReadHTTPRequestTraceAsSpan(record *ringbuf.Record) (request.Span, bool, error) {
   var event HTTPRequestTrace
  
   // 将ringbuf.Record转换为HTTPRequestTrace
   err := binary.Read(bytes.NewBuffer(record.RawSample), binary.LittleEndian, &event)
   if err != nil {
      return request.Span{}, true, err
   }
   // 将HTTPRequestTrace转换为request.Span
   return HTTPRequestTraceToSpan(&event), false, nil
  }

go中的HTTPRequestTrace结构定义如下：

• 与ebpf中的http_request_trace结构定义一致；

// pkg/internal/ebpf/common/common.go

//go:generate $BPF2GO -cc $BPF_CLANG -cflags $BPF_CFLAGS -target bpf -type http_request_trace bpf ../../../../bpf/http_trace.c -- -I../../../../bpf/headers

// HTTPRequestTrace contains information from an HTTP request as directly received from the
// eBPF layer. This contains low-level C structures for accurate binary read from ring buffer.
type HTTPRequestTrace bpfHttpRequestTrace
// pkg/internal/ebpf/common/bpf_bpf.go
type bpfHttpRequestTrace struct {
    Type              uint8
    Id                uint64
    GoStartMonotimeNs uint64
    StartMonotimeNs   uint64
    EndMonotimeNs     uint64
    Method            [7]uint8
    Path              [100]uint8
    Status            uint16
    RemoteAddr        [50]uint8
    RemoteAddrLen     uint64
    Host              [256]uint8
    HostLen           uint64
    HostPort          uint32
    ContentLength     int64
    Traceparent       [55]uint8
}

五. 将HTTPRequestTrace转换为request.Span

由HTTPRequestTraceToSpan()函数负责：

// pkg/internal/ebpf/common/common.go
func HTTPRequestTraceToSpan(trace *HTTPRequestTrace) request.Span {
    ...
    switch request.EventType(trace.Type) {
    case request.EventTypeHTTPClient, request.EventTypeHTTP:
       peer, _ = extractHostPort(trace.RemoteAddr[:])
       hostname, hostPort = extractHostPort(trace.Host[:])
    case request.EventTypeGRPC:
       hostPort = int(trace.HostPort)
       peer = extractIP(trace.RemoteAddr[:], int(trace.RemoteAddrLen))
       hostname = extractIP(trace.Host[:], int(trace.HostLen))
    case request.EventTypeGRPCClient:
       hostname, hostPort = extractHostPort(trace.Host[:])
    default:
       log.Warn("unknown trace type", "type", trace.Type)
    }

    return request.Span{
       Type:          request.EventType(trace.Type),
       ID:            trace.Id,
       Method:        string(trace.Method[:methodLen]),
       Path:          string(trace.Path[:pathLen]),
       Peer:          peer,
       Host:          hostname,
       HostPort:      hostPort,
       ContentLength: trace.ContentLength,
       RequestStart:  int64(trace.GoStartMonotimeNs),
       Start:         int64(trace.StartMonotimeNs),
       End:           int64(trace.EndMonotimeNs),
       Status:        int(trace.Status),
       Traceparent:   traceparent,
    }
}

六. metricReporter将request.Span转换为metric指标

// pkg/internal/export/prom/prom.go
func (r *metricsReporter) reportMetrics(input <-chan []request.Span) {
   go r.promConnect.StartHTTP(r.bgCtx)
   for spans := range input {
      for i := range spans {
         r.observe(&spans[i])
      }
   }
}

func (r *metricsReporter) observe(span *request.Span) {
   t := span.Timings()
   duration := t.End.Sub(t.RequestStart).Seconds()
   switch span.Type {
   case request.EventTypeHTTP:
      lv := r.labelValuesHTTP(span)
      r.httpDuration.WithLabelValues(lv...).Observe(duration)
      r.httpRequestSize.WithLabelValues(lv...).Observe(float64(span.ContentLength))
   case request.EventTypeHTTPClient:
      lv := r.labelValuesHTTPClient(span)
      r.httpClientDuration.WithLabelValues(lv...).Observe(duration)
      r.httpClientRequestSize.WithLabelValues(lv...).Observe(float64(span.ContentLength))
   case request.EventTypeGRPC:
      r.grpcDuration.WithLabelValues(r.labelValuesGRPC(span)...).Observe(duration)
   case request.EventTypeGRPCClient:
      r.grpcClientDuration.WithLabelValues(r.labelValuesGRPC(span)...).Observe(duration)
   }
}