Go1.10以上版本客户端使用https代理遇到oversized record received解决办法

Go1.10以上版本客户端使用https代理遇到oversized record received解决办法

最近在项目开发过程中需要做http回调，由于内网防火墙限制机器的外网访问权限，因此只能使用代理出口的方式进行访问第三方的回调接口，由于公司内部有一台https的代理服务器因此打算用此代理服务。在使用 Go 1.10以上版本的时候，Go客户端遇到下问题proxyconnect tcp: tls: oversized record received with length 20527 ，各大搜索引擎都没有找到解决办法，经过几个小时的试错后，居然发现Go1.8版本居然没有这个问题，因此果断对比Go1.8和Go1.10的源码，并在源码中加Debug信息，后面找了出现问题的代码，具体解决办法见如下博文。

具体部署如下

httpclient   ---->   https-proxy ---->  第三方http接口

httpclient代码如下

package main

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/cihub/seelog"
)

func PostHttpRequest(address, data string, hdrs map[string]string) (string, error) {
	code, body, err := RawPostHttpRequest(address, data, hdrs, map[string]string{})
	if code != 200 {
		seelog.Errorf("post request error %d %s %s %s\n", code, data, body, err)
		return "", err
	}

	return body, nil
}

func PostHttpRequestEx(address, data string, hdrs, config map[string]string) (string, error) {
	code, body, err := RawPostHttpRequest(address, data, hdrs, config)
	if code != 200 {
		seelog.Errorf("post request error %d %s %s %s\n", code, data, body, err)
		return "", err
	}

	return body, nil
}

func RawPostHttpRequest(address, data string, hdrs, configs map[string]string) (int, string, error) {
	var proxy func(*http.Request) (*url.URL, error) = nil
	if v := configs["proxy"]; v != "" {
		proxy = func(req *http.Request) (*url.URL, error) {
			return url.Parse(v)
		}
	}

	client := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},

			Proxy: proxy,
		},

		Timeout: time.Duration(6) * time.Second,
	}

	req, err := http.NewRequest(
		"POST",
		address,
		strings.NewReader(data),
	)
	if err != nil {
		// handle error
		seelog.Error("new post request error ", err, data)
		return -1, "", err
	}

	if hdrs != nil {
		for k, v := range hdrs {
			if k == "Host" {
				req.Host = v
				continue
			}

			req.Header[k] = []string{v}
		}
	}

	if hdrs == nil || hdrs["Content-Type"] == "" {
		req.Header.Set("Content-Type", "application/json")
	}

	resp, err := client.Do(req)
	if err != nil {
		seelog.Error("post request error ", err, data)
		return -1, "", err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		// handle error
		seelog.Error("post request error ", err, data)
		return -1, "", err
	}

	if resp.StatusCode != 200 {
		err = fmt.Errorf(resp.Status)
	}

	return resp.StatusCode, string(body), err
}

func main() {
	defer seelog.Flush()

	curl := ""
	if len(os.Args) >= 2 {
		curl = os.Args[1]
	}

	cfg := map[string]string{}
	if len(os.Args) >= 3 {
		cfg["proxy"] = os.Args[2]
	}

	data := "{}"
	seelog.Debugf("callback cfg %v", cfg)

	body, err := PostHttpRequestEx(curl, data, nil, cfg)
	if err != nil {
		seelog.Errorf("callback request error %s", err)
		return
	}

	seelog.Infof("callback %s post %s => %s", curl, data, body)
}

编译运行出现如下错误

taxue@linux:~$ go build HttpClient.go
taxue@linux:~$ ./HttpClient https://www.baidu.com https://proxy.server.com:1101
1550890998458292103 [Debug] callback cfg map[proxy:https://proxy.server.com:1101]
1550890998467744401 [Error] post request error Post https://www.baidu.com: proxyconnect tcp: tls: oversized record received with length 20527{}
1550890998467789535 [Error] callback request error Post https://www.baidu.com: proxyconnect tcp: tls: oversized record received with length 20527

问题原因

在/usr/local/go/src/net/http/transport.go文件中dialConn函数，Go1.10版本以后加了if cm.scheme() == "https" { 这个条件判断，错误就是在这个条件块里面出现的，而Go1.8是没有这段代码的，因此猜想可能是某些原因导致加入了这8行代码导致https代理失效。

func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (*persistConn, error) {
	if cm.scheme() == "https" && t.DialTLS != nil {
		...
	} else {
		conn, err := t.dial(ctx, "tcp", cm.addr())
		if err != nil {
			return nil, wrapErr(err)
		}
		pconn.conn = conn
		if cm.scheme() == "https" {
			var firstTLSHost string
			if firstTLSHost, _, err = net.SplitHostPort(cm.addr()); err != nil {
				return nil, wrapErr(err)
			}
			if err = pconn.addTLS(firstTLSHost, trace); err != nil {
				return nil, wrapErr(err)
			}
		}
	}

解决办法

改进办法就是添加条件，当为代理模式的时候不进入这个条件。

if cm.scheme() == "https" && cm.proxyURL == nil {
			var firstTLSHost string
			if firstTLSHost, _, err = net.SplitHostPort(cm.addr()); err != nil {
				return nil, wrapErr(err)
			}
			if err = pconn.addTLS(firstTLSHost, trace); err != nil {
				return nil, wrapErr(err)
			}
		}

taxue@linux:~$ go build HttpClient.go
taxue@linux:~$ ./HttpClient https://www.baidu.com  https://zmcc.corp.cootek.com:1101
1550895140686682645 [Debug] callback cfg map[proxy:https://zmcc.corp.cootek.com:1101]
1550895140899872758 [Info] callback https://www.baidu.com post {} => <!DOCTYPE html>
<!--STATUS OK-->
<html>
<head>
    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
    <meta http-equiv="content-type" content="text/html;charset=utf-8">
    <meta content="always" name="referrer">
    <script src="https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/nocache/imgdata/seErrorRec.js"></script>
    <title>页面不存在_百度搜索</title>
......

总结

至此测试了 http、https直接访问，https代理访问http、https都正常了。是不是golang官方的bug不得而知，欢迎各路大神拍砖过来。