golang http client为什么要close Response.Body,必须close吗?

我们知道一般在调用http client后都会close Response.Body,如下:

client := http.DefaultClient
resp, err := client.Do(req)
if err != nil {
    return nil, err
}
defer resp.Body.Close()

下面我们来看下为什么resp.Body需要Close,一定需要Close吗？

我们先通过"net/http/httptrace"来验证下:

1.不使用Close

代码:

package main

import (
    "fmt"
    "net/http"
    "net/http/httptrace"
)

func main() {
    req, err := http.NewRequest("GET", "https://www.baidu.com/", nil)
    if err != nil {
        panic(err)
    }

    trace := &httptrace.ClientTrace{
        GetConn: func(hostPort string) {
            fmt.Println("GetConn:", hostPort)
        },
        GotConn: func(connInfo httptrace.GotConnInfo) {
            fmt.Printf("GotConn Reused=%v WasIdle=%v IdleTime=%v\n", connInfo.Reused, connInfo.WasIdle, connInfo.IdleTime)
            fmt.Printf("GotConn localAddr: %s\n", connInfo.Conn.LocalAddr())
            fmt.Printf("GotConn remoteAddr: %s\n", connInfo.Conn.RemoteAddr())
        },
        PutIdleConn: func(err error) {
            fmt.Printf("PutIdleConn err=%v\n", err)
        },
    }

    req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))

    client := &http.Client{
        Transport: &http.Transport{
            MaxIdleConns:    1, //最大Idle
            MaxConnsPerHost: 2, //每个host最大conns
        }}

    for i := 1; i <= 10; i++ {
        fmt.Printf("==============第[%d]次请求================\n", i)
        resp, err := client.Do(req)
        if err != nil {
            fmt.Println(err)
            return
        }
        fmt.Println(resp.Status)
    }
}

输出:

==============第[1]次请求================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:55131
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============第[2]次请求================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:55132
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============第[3]次请求================
GetConn: www.baidu.com:443

结论:
我们设置了MaxConnsPerHost=2,由于没有close导致没有释放连接，执行两次请求后就卡住了，不能继续向下执行。并且第一次和第二次请求连接没有复用

2.只使用Close,不读取resp.Body

代码:

package main

import (
    "fmt"
    "net/http"
    "net/http/httptrace"
)

func main() {
    req, err := http.NewRequest("GET", "https://www.baidu.com/", nil)
    if err != nil {
        panic(err)
    }

    trace := &httptrace.ClientTrace{
        GetConn: func(hostPort string) {
            fmt.Println("GetConn:", hostPort)
        },
        GotConn: func(connInfo httptrace.GotConnInfo) {
            fmt.Printf("GotConn Reused=%v WasIdle=%v IdleTime=%v\n", connInfo.Reused, connInfo.WasIdle, connInfo.IdleTime)
            fmt.Printf("GotConn localAddr: %s\n", connInfo.Conn.LocalAddr())
            fmt.Printf("GotConn remoteAddr: %s\n", connInfo.Conn.RemoteAddr())
        },
        PutIdleConn: func(err error) {
            fmt.Printf("PutIdleConn err=%v\n", err)
        },
    }

    req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))

    client := &http.Client{
        Transport: &http.Transport{
            MaxIdleConns:    1, //最大Idle
            MaxConnsPerHost: 2, //每个host最大conns
        }}

    for i := 1; i <= 10; i++ {
        fmt.Printf("==============第[%d]次请求================\n", i)
        resp, err := client.Do(req)
        if err != nil {
            fmt.Println(err)
            return
        }
        resp.Body.Close() //注意这里close了
        fmt.Println(resp.Status)
    }
}

输出:

==============1================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54948
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============2================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54949
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============3================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54950
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============4================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54954
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============5================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54955
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============6================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54956
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============7================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54957
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============8================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54958
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============9================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54959
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============10================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54960
GotConn remoteAddr: 183.232.231.172:443
200 OK

结论：
虽然可以继续执行，说明连接释放了。但GotConn信息eused=false WasIdle=false并且每次localAddr都是不同的,我们发现每次请求获得的连接都是新申请的。

3.只读取resp.Body,不使用Close()

我们知道resp.Body实现了io.Reader接口方法,我们通常的做法就通过调用Read()方法来读取内容。
代码:

package main

import (
    "fmt"
    "io"
    "net/http"
    "net/http/httptrace"
)

func main() {
    req, err := http.NewRequest("GET", "https://www.baidu.com/", nil)
    if err != nil {
        panic(err)
    }

    trace := &httptrace.ClientTrace{
        GetConn: func(hostPort string) {
            fmt.Println("GetConn:", hostPort)
        },
        GotConn: func(connInfo httptrace.GotConnInfo) {
            fmt.Printf("GotConn Reused=%v WasIdle=%v IdleTime=%v\n", connInfo.Reused, connInfo.WasIdle, connInfo.IdleTime)
            fmt.Printf("GotConn localAddr: %s\n", connInfo.Conn.LocalAddr())
            fmt.Printf("GotConn remoteAddr: %s\n", connInfo.Conn.RemoteAddr())
        },
        PutIdleConn: func(err error) {
            fmt.Printf("PutIdleConn err=%v\n", err)
        },
    }

    req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))

    client := &http.Client{
        Transport: &http.Transport{
            MaxIdleConns:    1, //最大Idle
            MaxConnsPerHost: 2, //每个host最大conns
        }}

    for i := 1; i <= 10; i++ {
        fmt.Printf("==============第[%d]次请求================\n", i)
        resp, err := client.Do(req)
        if err != nil {
            fmt.Println(err)
            return
        }
        io.ReadAll(resp.Body) //读取body里面的内容
        fmt.Println(resp.Status)
    }
}

输出:

==============第[1]次请求================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[2]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=116.037µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[3]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=72.154µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[4]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=60.102µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[5]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=71.807µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[6]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=74.616µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[7]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=47.205µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[8]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=74.207µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[9]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=52.414µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK
==============第[10]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=81.137µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=<nil>
200 OK

结论:
我们可以看下每次输出的GotConn信息Reused=true WasIdle=true并且localAddr都是相同的。说明每次执行完后都把连接放回了idleConn pool里面，下去获取连接时可以从idleConn pool里面获取。

源码分析

(c *Client) Do(req *Request)-->(t *Transport) roundTrip(req *Request)-->t.getConn(treq, cm)-->t.queueForDial(w)->go t.dialConnFor(w)-->go readLoop()
我们readLoop()源码来分析，readLoop()是循环读取连接内容的方法:

go1.18 net/http/transport.go
(pc *persistConn) readLoop()是一个持久化连接在不停的读取conn里面的内容，下面我跟随源码看一下为什么要close/readAll

func (pc *persistConn) readLoop() {
    closeErr := errReadLoopExiting // default value, if not changed below
    defer func() {
        pc.close(closeErr)
        pc.t.removeIdleConn(pc)
    }()

    // tryPutIdleConn就是把连接放回IdleConn然后让连接可以复用
    tryPutIdleConn := func(trace *httptrace.ClientTrace) bool {
        if err := pc.t.tryPutIdleConn(pc); err != nil {
            closeErr = err
            if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled {
                trace.PutIdleConn(err)
            }
            return false
        }
        if trace != nil && trace.PutIdleConn != nil {
            trace.PutIdleConn(nil)
        }
        return true
    }

    // eofc is used to block caller goroutines reading from Response.Body
    // at EOF until this goroutines has (potentially) added the connection
    // back to the idle pool.
    eofc := make(chan struct{})
    defer close(eofc) // unblock reader on errors

    // Read this once, before loop starts. (to avoid races in tests)
    testHookMu.Lock()
    testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead
    testHookMu.Unlock()

     //这里的alive用来判断是否继续读取连接内容
    alive := true
    for alive {
        pc.readLimit = pc.maxHeaderResponseSize()
        _, err := pc.br.Peek(1)

        pc.mu.Lock()
        if pc.numExpectedResponses == 0 {
            pc.readLoopPeekFailLocked(err)
            pc.mu.Unlock()
            return
        }
        pc.mu.Unlock()

        rc := <-pc.reqch
        trace := httptrace.ContextClientTrace(rc.req.Context())

        var resp *Response
        if err == nil {
            resp, err = pc.readResponse(rc, trace)
        } else {
            err = transportReadFromServerError{err}
            closeErr = err
        }

        if err != nil {
            if pc.readLimit <= 0 {
                err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize())
            }

            select {
            case rc.ch <- responseAndError{err: err}:
            case <-rc.callerGone:
                return
            }
            return
        }
        pc.readLimit = maxInt64 // effectively no limit for response bodies

        pc.mu.Lock()
        pc.numExpectedResponses--
        pc.mu.Unlock()

        bodyWritable := resp.bodyIsWritable()
        hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0
        
        //注意这里如果bodyWritable=false会把alive设置成false
        if resp.Close || rc.req.Close || resp.StatusCode <= 199 || bodyWritable {
            // Don't do keep-alive on error if either party requested a close
            // or we get an unexpected informational (1xx) response.
            // StatusCode 100 is already handled above.
            alive = false
        }

        // 这里处理了没有body和body可以被写入的情况,不做过多讨论,可跳过
        if !hasBody || bodyWritable {
            replaced := pc.t.replaceReqCanceler(rc.cancelKey, nil)

            // Put the idle conn back into the pool before we send the response
            // so if they process it quickly and make another request, they'll
            // get this same conn. But we use the unbuffered channel 'rc'
            // to guarantee that persistConn.roundTrip got out of its select
            // potentially waiting for this persistConn to close.
            //如果没有没有body会直接回收conn,执行tryPutIdleConn(),
            //和bodyWritable没关系,前面如果是bodyWritable会把alive设置成false,就不会执行tryPutIdleConn()
            alive = alive &&
                !pc.sawEOF &&
                pc.wroteRequest() &&
                replaced && tryPutIdleConn(trace)

            if bodyWritable {
                closeErr = errCallerOwnsConn
            }

            select {
            case rc.ch <- responseAndError{res: resp}:
            case <-rc.callerGone:
                return
            }

            // Now that they've read from the unbuffered channel, they're safely
            // out of the select that also waits on this goroutine to die, so
            // we're allowed to exit now if needed (if alive is false)
            testHookReadLoopBeforeNextRead()
            continue
        }

        waitForBodyRead := make(chan bool, 2)
        //最重要的就是这个把resp.body通过bodyEOFSignal封装生成新的resp.Body，
        //下面会讲到为什么通过bodyEOFSignal封装
        body := &bodyEOFSignal{
            body: resp.Body,
            //如果调用earlyCloseFn就执行waitForBodyRead <- false
            earlyCloseFn: func() error {
                waitForBodyRead <- false
                <-eofc // will be closed by deferred call at the end of the function
                return nil

            },
            //如果调用fn()并且 err == io.EOF 就执行waitForBodyRead <- true
            fn: func(err error) error {
                isEOF := err == io.EOF
                waitForBodyRead <- isEOF
                if isEOF {
                    <-eofc // see comment above eofc declaration
                } else if err != nil {
                    if cerr := pc.canceled(); cerr != nil {
                        return cerr
                    }
                }
                return err
            },
        }

        // 重新把封装成bodyEOFSignal的body赋值给resp.Body
        resp.Body = body
        if rc.addedGzip && ascii.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {
            resp.Body = &gzipReader{body: body}
            resp.Header.Del("Content-Encoding")
            resp.Header.Del("Content-Length")
            resp.ContentLength = -1
            resp.Uncompressed = true
        }

        select {
        // 把resp推送到rc.ch然后在roundTrip()返回给外部
        case rc.ch <- responseAndError{res: resp}:
        case <-rc.callerGone:
            return
        }

        // Before looping back to the top of this function and peeking on
        // the bufio.Reader, wait for the caller goroutine to finish
        // reading the response body. (or for cancellation or death)
        select {
        // 这里是最重要的,从waitForBodyRead阻塞获取bodyEof
        case bodyEOF := <-waitForBodyRead:
            replaced := pc.t.replaceReqCanceler(rc.cancelKey, nil) // before pc might return to idle pool
            //这里比较巧妙的是如果alive && bodyEOF && !pc.sawEOF && pc.wroteRequest()
            //才会执行tryPutIdleConn(trace),就是把连接放回idleConn,这就是为什么使用close()是关闭了连接但是没有复用，而使用ReadAll()确复用了idle。
            //因为使用使用ReadAll后返回的bodyEOF=true而直接使用Close()返回的bodyEOF=false导致永远不会执行tryPutIdleConn(trace)。
            //注意pc.sawEOF和body Close没关系，这个是检测conn是否Close
            alive = alive &&
                bodyEOF &&
                !pc.sawEOF &&
                pc.wroteRequest() &&
                replaced && tryPutIdleConn(trace)
            if bodyEOF {
                eofc <- struct{}{}
            }
        case <-rc.req.Cancel:
            alive = false
            pc.t.CancelRequest(rc.req)
        case <-rc.req.Context().Done():
            alive = false
            pc.t.cancelRequest(rc.cancelKey, rc.req.Context().Err())
        case <-pc.closech:
            alive = false
        }

        testHookReadLoopBeforeNextRead()
    }
}

下面我们来看下resp.Body的结构bodyEOFSignal:

//readLoop()里面的body
        body := &bodyEOFSignal{
            body: resp.Body,
             //如果调用earlyCloseFn就执行waitForBodyRead <- false
            earlyCloseFn: func() error {
                waitForBodyRead <- false
                <-eofc // will be closed by deferred call at the end of the function
                return nil

            },
            //如果调用fn()并且 err == io.EOF 就执行waitForBodyRead <- true
            fn: func(err error) error {
                isEOF := err == io.EOF
                waitForBodyRead <- isEOF
                if isEOF {
                    <-eofc // see comment above eofc declaration
                } else if err != nil {
                    if cerr := pc.canceled(); cerr != nil {
                        return cerr
                    }
                }
                return err
            },
        }

// bodyEOFSignal is used by the HTTP/1 transport when reading response
// bodies to make sure we see the end of a response body before
// proceeding and reading on the connection again.
//
// It wraps a ReadCloser but runs fn (if non-nil) at most
// once, right before its final (error-producing) Read or Close call
// returns. fn should return the new error to return from Read or Close.
//
// If earlyCloseFn is non-nil and Close is called before io.EOF is
// seen, earlyCloseFn is called instead of fn, and its return value is
// the return value from Close.
type bodyEOFSignal struct {
    body         io.ReadCloser
    mu           sync.Mutex        // guards following 4 fields
    closed       bool              // whether Close has been called
    rerr         error             // sticky Read error
    fn           func(error) error // err will be nil on Read io.EOF
    earlyCloseFn func() error      // optional alt Close func used if io.EOF not seen 如果没EOF就会被调用
}

func (es *bodyEOFSignal) Read(p []byte) (n int, err error) {
    es.mu.Lock()
    closed, rerr := es.closed, es.rerr
    es.mu.Unlock()
    if closed {
        return 0, errReadOnClosedResBody
    }
    if rerr != nil {
        return 0, rerr
    }

    n, err = es.body.Read(p)
    if err != nil {
        es.mu.Lock()
        defer es.mu.Unlock()
        if es.rerr == nil {
            es.rerr = err
        }
        //condfn会去执行es.fn
        err = es.condfn(err)
    }
    return
}

func (es *bodyEOFSignal) Close() error {
    es.mu.Lock()
    defer es.mu.Unlock()
    if es.closed {
        return nil
    }
    es.closed = true
    // 如果调用Close的时候没有EOF就会调用earlyCloseFn() 
    if es.earlyCloseFn != nil && es.rerr != io.EOF {
        return es.earlyCloseFn() 
    }
    err := es.body.Close()
    return es.condfn(err)
}

// caller must hold es.mu.
// 把err传给es.fn, es.fn会通过err做不同的操作，主要是判断err是不是EOF
func (es *bodyEOFSignal) condfn(err error) error {
    if es.fn == nil {
        return err
    }
    err = es.fn(err) 
    es.fn = nil
    return err
}

这个pr增加了注释说明了如果不执行Read()在http1.x是不会重用连接的：
https://go-review.googlesource.com/c/go/+/1531

总结

我们收到的resp.body是bodyEOFSignal,如果不执行Close()会导致readLoop()阻塞。此时如果设置了最大连接MaxConnsPerHost则达到连接数后不能再申请。但是如果只是Close()而不执行Read()则会导致连接不会放回idleConn

推荐用法:
不一定要执行Close(),但一定要执行Read():
如不需要body里面的内容可以执行

client := http.DefaultClient
resp, err := client.Do(req)
if err != nil {
    return nil, err
}
defer io.Copy(io.Discard,resp.Body)

如果需要resp.Body则执行:

client := http.DefaultClient
resp, err := client.Do(req)
if err != nil {
    return nil, err
}
//这里的resp.Body.Close()是防止Read操作不能被正确执行到做兜底,
//如果能够确保io.ReadAll(resp.Body)被执行到也不需要Close()
defer resp.Body.Close() 
//读取resp.Body,如io.ReadAll/io.Copy()...
io.ReadAll(resp.Body)