From fe79fe9fc73962928f1d0f16eb53e95c5abaa6d2 Mon Sep 17 00:00:00 2001 From: libotony Date: Mon, 19 Aug 2019 21:46:27 +0800 Subject: [PATCH 01/34] get real ip of http request --- web/controllers/login.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/controllers/login.go b/web/controllers/login.go index c31e9a1..64873a4 100755 --- a/web/controllers/login.go +++ b/web/controllers/login.go @@ -22,7 +22,7 @@ func (self *LoginController) Verify() { if self.GetString("password") == beego.AppConfig.String("web_password") && self.GetString("username") == beego.AppConfig.String("web_username") { self.SetSession("isAdmin", true) auth = true - server.Bridge.Register.Store(common.GetIpByAddr(self.Ctx.Request.RemoteAddr), time.Now().Add(time.Hour*time.Duration(2))) + server.Bridge.Register.Store(common.GetIpByAddr(self.Ctx.Input.IP()), time.Now().Add(time.Hour*time.Duration(2))) } b, err := beego.AppConfig.Bool("allow_user_login") if err == nil && b && !auth { From 89df38422c3d5c33ef2d14f5dfa7892b7bf9892c Mon Sep 17 00:00:00 2001 From: he liu Date: Mon, 2 Sep 2019 16:18:03 +0800 Subject: [PATCH 02/34] Update bridge.go --- bridge/bridge.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridge/bridge.go b/bridge/bridge.go index 3247dd8..3f2c3a8 100755 --- a/bridge/bridge.go +++ b/bridge/bridge.go @@ -296,7 +296,7 @@ func (s *Bridge) register(c *conn.Conn) { func (s *Bridge) SendLinkInfo(clientId int, link *conn.Link, t *file.Tunnel) (target net.Conn, err error) { //if the proxy type is local if link.LocalProxy { - target, err = net.Dial(link.ConnType, link.Host) + target, err = net.Dial("tcp", link.Host) return } if v, ok := s.Client.Load(clientId); ok { From fce53fa308256781794aa875f1e822968097ec50 Mon Sep 17 00:00:00 2001 From: zhangwei Date: Tue, 10 Sep 2019 23:14:05 +0800 Subject: [PATCH 03/34] new feature multi user auth with socks5 --- conf/multiuser.conf | 0 go.mod | 9 +++------ go.sum | 24 ++++++++---------------- lib/config/config.go | 26 ++++++++++++++++++++++++++ lib/file/obj.go | 10 ++++++---- npc.conf | 21 +++++++++++++++++++++ server/proxy/socks5.go | 19 +++++++++++++++++-- 7 files changed, 81 insertions(+), 28 deletions(-) create mode 100644 conf/multiuser.conf create mode 100644 npc.conf diff --git a/conf/multiuser.conf b/conf/multiuser.conf new file mode 100644 index 0000000..e69de29 diff --git a/go.mod b/go.mod index 1f6b753..e8111a1 100644 --- a/go.mod +++ b/go.mod @@ -5,23 +5,20 @@ go 1.12 require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/astaxie/beego v1.12.0 - github.com/belogik/goes v0.0.0-20151229125003-e54d722c3aff // indirect github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d github.com/go-ole/go-ole v1.2.4 // indirect github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db github.com/klauspost/cpuid v1.2.1 // indirect - github.com/klauspost/reedsolomon v1.9.2 - github.com/onsi/gomega v1.5.0 // indirect + github.com/klauspost/reedsolomon v1.9.2 // indirect github.com/pkg/errors v0.8.0 github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 // indirect github.com/shirou/gopsutil v2.18.12+incompatible github.com/stretchr/testify v1.3.0 // indirect github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161 // indirect - github.com/templexxx/xor v0.0.0-20181023030647-4e92f724b73b - github.com/tjfoc/gmsm v1.0.1 + github.com/templexxx/xor v0.0.0-20181023030647-4e92f724b73b // indirect + github.com/tjfoc/gmsm v1.0.1 // indirect github.com/xtaci/kcp-go v5.4.4+incompatible github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae // indirect - golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa // indirect ) diff --git a/go.sum b/go.sum index 29de132..a0396f9 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,9 @@ github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OwnLocal/goes v1.0.0/go.mod h1:8rIFjBGTue3lCU0wplczcUgt9Gxgrkkrw7etMIcn8TM= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/astaxie/beego v1.12.0 h1:MRhVoeeye5N+Flul5PoVfD9CslfdoH+xqC/xvSQ5u2Y= -github.com/astaxie/beego v1.12.0/go.mod h1:fysx+LZNZKnvh4GED/xND7jWtjCR6HzydR2Hh2Im57o= github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd/go.mod h1:1b+Y/CofkYwXMUU0OhQqGvsY2Bvgr4j6jfT699wyZKQ= github.com/beego/x2j v0.0.0-20131220205130-a0352aadc542/go.mod h1:kSeGC/p1AbBiEp5kat81+DSQrZenVBZXklMLaELspWU= -github.com/belogik/goes v0.0.0-20151229125003-e54d722c3aff/go.mod h1:PhH1ZhyCzHKt4uAasyx+ljRCgoezetRNf59CUtwUkqY= github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= github.com/casbin/casbin v1.7.0/go.mod h1:c67qKN6Oum3UF5Q1+BByfFxkwKvhwW57ITjqwtzR1KE= github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d h1:As4937T5NVbJ/DmZT9z33pyLEprMd6CUSfhbmMY57Io= @@ -15,34 +13,31 @@ github.com/couchbase/go-couchbase v0.0.0-20181122212707-3e9b6e1258bb/go.mod h1:T github.com/couchbase/gomemcached v0.0.0-20181122193126-5125a94a666c/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c= github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs= github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGiisLwp9rITslkFNpZD5rz43tf41QFkTWY= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elazarl/go-bindata-assetfs v1.0.0 h1:G/bYguwHIzWq9ZoyUQqrjTmJbbYn3j3CKKpKinvZLFk= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/exfly/beego v1.12.0-export-init h1:VQNYKdXhAwZGUaFmQv8Aj921O3rQJZRIF8xeGrhsjrI= github.com/exfly/beego v1.12.0-export-init/go.mod h1:fysx+LZNZKnvh4GED/xND7jWtjCR6HzydR2Hh2Im57o= -github.com/exfly/beego v1.12.0 h1:OXwIwngaAx35Mga+jLiZmArusBxj8/H0jYXzGDAdwOg= -github.com/exfly/beego v1.12.0/go.mod h1:fysx+LZNZKnvh4GED/xND7jWtjCR6HzydR2Hh2Im57o= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/reedsolomon v1.9.2 h1:E9CMS2Pqbv+C7tsrYad4YC9MfhnMVWhMRsTi7U0UB18= github.com/klauspost/reedsolomon v1.9.2/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 h1:X+yvsM2yrEktyI+b2qND5gpH8YhURn0k8OCaeRnkINo= github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644/go.mod h1:nkxAfR/5quYxwPZhyDxgasBMnRtBZd0FCEpawpjMUFg= @@ -53,6 +48,7 @@ github.com/siddontang/ledisdb v0.0.0-20181029004158-becf5f38d373/go.mod h1:mF1Dp github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d/go.mod h1:AMEsy7v5z92TR1JKMkLLoaOQk++LVnOKL3ScbJ8GNGA= github.com/ssdb/gossdb v0.0.0-20180723034631-88f6b59b84ec/go.mod h1:QBvMkMya+gXctz3kmljlUCu/yB3GZ6oee+dUozsezQE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161 h1:89CEmDvlq/F7SJEOqkIdNDGJXrQIhuIx9D2DBXjavSU= @@ -64,19 +60,15 @@ github.com/tjfoc/gmsm v1.0.1/go.mod h1:XxO4hdhhrzAd+G4CjDqaOkd0hUzmtPR/d3EiBBMn/ github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc= github.com/xtaci/kcp-go v5.4.4+incompatible h1:QIJ0a0Q0N1G20yLHL2+fpdzyy2v/Cb3PI+xiwx/KK9c= github.com/xtaci/kcp-go v5.4.4+incompatible/go.mod h1:bN6vIwHQbfHaHtFpEssmWsN45a+AZwO7eyRCmEIbtvE= +github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae h1:J0GxkO96kL4WF+AIT3M4mfUVinOCPgf2uUWYFUzN0sM= github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE= golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85 h1:et7+NAX3lLIk5qUCTA9QelBjGE/NkhzYw/mhnr0s7nI= golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa h1:KIDDMLT1O0Nr7TSxp8xM5tJcdn8tgyAONntO829og1M= golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/lib/config/config.go b/lib/config/config.go index c35afb7..d66618f 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -239,12 +239,38 @@ func dealTunnel(s string) *file.Tunnel { t.LocalPath = item[1] case "strip_pre": t.StripPre = item[1] + case "multi_user": + // TODO 解析多账号配置文件 + t.Client.Cnf.MultiUser = true + if b, err := common.ReadAllFromFile(item[1]); err != nil { + panic(err) + } else { + if content, err := common.ParseStr(string(b)); err != nil { + panic(err) + } else { + t.Client.Cnf.MultiUserMap = dealMultiUser(content) + } + } } } return t } +func dealMultiUser(s string) map[string]string { + multiUserMap := make(map[string]string) + for _, v := range splitStr(s) { + item := strings.Split(v, "=") + if len(item) == 0 { + continue + } else if len(item) == 1 { + item = append(item, "") + } + multiUserMap[strings.TrimSpace(item[0])] = item[1] + } + return multiUserMap +} + func delLocalService(s string) *LocalServer { l := new(LocalServer) for _, v := range splitStr(s) { diff --git a/lib/file/obj.go b/lib/file/obj.go index d3a1fbe..2215968 100644 --- a/lib/file/obj.go +++ b/lib/file/obj.go @@ -25,10 +25,12 @@ func (s *Flow) Add(in, out int64) { } type Config struct { - U string - P string - Compress bool - Crypt bool + U string + P string + Compress bool + Crypt bool + MultiUser bool // enable multi user authentication. + MultiUserMap map[string]string // multi user and pwd } type Client struct { diff --git a/npc.conf b/npc.conf new file mode 100644 index 0000000..63a8a37 --- /dev/null +++ b/npc.conf @@ -0,0 +1,21 @@ +[common] +server_addr=proxy.njcp.info:55000 +conn_type=tcp +vkey=%EEKn6q386JmCQC^ +auto_reconnection=true +max_conn=1000 +flow_limit=1000 +rate_limit=1000 +basic_username=bobanNanJing +basic_password=N^#xf407W%ZfOg$u +web_username=12 +web_password=12 +crypt=true +compress=true +username=bobanNanJing +password=0Ho@^#U&y571%cK* + + +[socks5] +mode=socks5 +server_port=19009 \ No newline at end of file diff --git a/server/proxy/socks5.go b/server/proxy/socks5.go index 2fe72c1..d6c6ec6 100755 --- a/server/proxy/socks5.go +++ b/server/proxy/socks5.go @@ -199,7 +199,7 @@ func (s *Sock5ModeServer) handleConn(c net.Conn) { c.Close() return } - if s.task.Client.Cnf.U != "" && s.task.Client.Cnf.P != "" { + if (s.task.Client.Cnf.U != "" && s.task.Client.Cnf.P != "") || (s.task.Client.Cnf.MultiUser && len(s.task.Client.Cnf.MultiUserMap) > 0) { buf[1] = UserPassAuth c.Write(buf) if err := s.Auth(c); err != nil { @@ -236,7 +236,22 @@ func (s *Sock5ModeServer) Auth(c net.Conn) error { if _, err := io.ReadAtLeast(c, pass, passLen); err != nil { return err } - if string(user) == s.task.Client.Cnf.U && string(pass) == s.task.Client.Cnf.P { + + var U, P string + if s.task.Client.Cnf.MultiUser { + // enable multi user auth + U = string(user) + var ok bool + P, ok = s.task.Client.Cnf.MultiUserMap[U] + if !ok { + return errors.New("验证不通过") + } + } else { + U = s.task.Client.Cnf.U + P = s.task.Client.Cnf.P + } + + if string(user) == U && string(pass) == P { if _, err := c.Write([]byte{userAuthVersion, authSuccess}); err != nil { return err } From 5e55b761cf29118937d7d403dcff9a40bdcba474 Mon Sep 17 00:00:00 2001 From: zhangwei Date: Tue, 10 Sep 2019 23:15:38 +0800 Subject: [PATCH 04/34] new feature multi user auth with socks5 --- lib/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/config/config.go b/lib/config/config.go index d66618f..9ab4b6c 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -240,7 +240,7 @@ func dealTunnel(s string) *file.Tunnel { case "strip_pre": t.StripPre = item[1] case "multi_user": - // TODO 解析多账号配置文件 + // TODO add test with multi user config file t.Client.Cnf.MultiUser = true if b, err := common.ReadAllFromFile(item[1]); err != nil { panic(err) From a05995fba5286fba6ccd77ce7a2e170016a1e487 Mon Sep 17 00:00:00 2001 From: zhangwei Date: Thu, 12 Sep 2019 08:22:12 +0800 Subject: [PATCH 05/34] new feature multi user auth with socks5 --- bridge/bridge.go | 1 + lib/config/config.go | 5 ++--- lib/file/obj.go | 15 +++++++++------ server/proxy/socks5.go | 6 +++--- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/bridge/bridge.go b/bridge/bridge.go index 3247dd8..0b613ab 100755 --- a/bridge/bridge.go +++ b/bridge/bridge.go @@ -488,6 +488,7 @@ loop: tl.Password = t.Password tl.LocalPath = t.LocalPath tl.StripPre = t.StripPre + tl.MultiUser = t.MultiUser if !client.HasTunnel(tl) { if err := file.GetDb().NewTask(tl); err != nil { logs.Notice("Add task error ", err.Error()) diff --git a/lib/config/config.go b/lib/config/config.go index 9ab4b6c..47290be 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -240,15 +240,14 @@ func dealTunnel(s string) *file.Tunnel { case "strip_pre": t.StripPre = item[1] case "multi_user": - // TODO add test with multi user config file - t.Client.Cnf.MultiUser = true + t.MultiUser = new(file.MultiUser) if b, err := common.ReadAllFromFile(item[1]); err != nil { panic(err) } else { if content, err := common.ParseStr(string(b)); err != nil { panic(err) } else { - t.Client.Cnf.MultiUserMap = dealMultiUser(content) + t.MultiUser.UserMap = dealMultiUser(content) } } } diff --git a/lib/file/obj.go b/lib/file/obj.go index 2215968..424b727 100644 --- a/lib/file/obj.go +++ b/lib/file/obj.go @@ -25,12 +25,10 @@ func (s *Flow) Add(in, out int64) { } type Config struct { - U string - P string - Compress bool - Crypt bool - MultiUser bool // enable multi user authentication. - MultiUserMap map[string]string // multi user and pwd + U string + P string + Compress bool + Crypt bool } type Client struct { @@ -142,6 +140,7 @@ type Tunnel struct { LocalPath string StripPre string Target *Target + MultiUser *MultiUser Health sync.RWMutex } @@ -186,6 +185,10 @@ type Target struct { sync.RWMutex } +type MultiUser struct { + UserMap map[string]string // multi user and pwd +} + func (s *Target) GetRandomTarget() (string, error) { if s.TargetArr == nil { s.TargetArr = strings.Split(s.TargetStr, "\n") diff --git a/server/proxy/socks5.go b/server/proxy/socks5.go index d6c6ec6..a195693 100755 --- a/server/proxy/socks5.go +++ b/server/proxy/socks5.go @@ -199,7 +199,7 @@ func (s *Sock5ModeServer) handleConn(c net.Conn) { c.Close() return } - if (s.task.Client.Cnf.U != "" && s.task.Client.Cnf.P != "") || (s.task.Client.Cnf.MultiUser && len(s.task.Client.Cnf.MultiUserMap) > 0) { + if (s.task.Client.Cnf.U != "" && s.task.Client.Cnf.P != "") || (s.task.MultiUser != nil && len(s.task.MultiUser.UserMap) > 0) { buf[1] = UserPassAuth c.Write(buf) if err := s.Auth(c); err != nil { @@ -238,11 +238,11 @@ func (s *Sock5ModeServer) Auth(c net.Conn) error { } var U, P string - if s.task.Client.Cnf.MultiUser { + if s.task.MultiUser != nil { // enable multi user auth U = string(user) var ok bool - P, ok = s.task.Client.Cnf.MultiUserMap[U] + P, ok = s.task.MultiUser.UserMap[U] if !ok { return errors.New("验证不通过") } From 11d185ad59c70b07f96234fb7ec945bd31b1ae6a Mon Sep 17 00:00:00 2001 From: zhangwei Date: Thu, 12 Sep 2019 22:54:53 +0800 Subject: [PATCH 06/34] new feature multi user auth with socks5 --- bridge/bridge.go | 2 +- lib/config/config.go | 6 +++--- lib/file/obj.go | 38 +++++++++++++++++++------------------- server/proxy/socks5.go | 6 +++--- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/bridge/bridge.go b/bridge/bridge.go index 0b613ab..2340e56 100755 --- a/bridge/bridge.go +++ b/bridge/bridge.go @@ -488,7 +488,7 @@ loop: tl.Password = t.Password tl.LocalPath = t.LocalPath tl.StripPre = t.StripPre - tl.MultiUser = t.MultiUser + tl.MultiAccount = t.MultiAccount if !client.HasTunnel(tl) { if err := file.GetDb().NewTask(tl); err != nil { logs.Notice("Add task error ", err.Error()) diff --git a/lib/config/config.go b/lib/config/config.go index 47290be..89a6bfd 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -239,15 +239,15 @@ func dealTunnel(s string) *file.Tunnel { t.LocalPath = item[1] case "strip_pre": t.StripPre = item[1] - case "multi_user": - t.MultiUser = new(file.MultiUser) + case "multi_account": + t.MultiAccount = &file.MultiAccount{} if b, err := common.ReadAllFromFile(item[1]); err != nil { panic(err) } else { if content, err := common.ParseStr(string(b)); err != nil { panic(err) } else { - t.MultiUser.UserMap = dealMultiUser(content) + t.MultiAccount.AccountMap = dealMultiUser(content) } } } diff --git a/lib/file/obj.go b/lib/file/obj.go index 424b727..15dea37 100644 --- a/lib/file/obj.go +++ b/lib/file/obj.go @@ -124,23 +124,23 @@ func (s *Client) HasHost(h *Host) bool { } type Tunnel struct { - Id int - Port int - ServerIp string - Mode string - Status bool - RunStatus bool - Client *Client - Ports string - Flow *Flow - Password string - Remark string - TargetAddr string - NoStore bool - LocalPath string - StripPre string - Target *Target - MultiUser *MultiUser + Id int + Port int + ServerIp string + Mode string + Status bool + RunStatus bool + Client *Client + Ports string + Flow *Flow + Password string + Remark string + TargetAddr string + NoStore bool + LocalPath string + StripPre string + Target *Target + MultiAccount *MultiAccount Health sync.RWMutex } @@ -185,8 +185,8 @@ type Target struct { sync.RWMutex } -type MultiUser struct { - UserMap map[string]string // multi user and pwd +type MultiAccount struct { + AccountMap map[string]string // multi account and pwd } func (s *Target) GetRandomTarget() (string, error) { diff --git a/server/proxy/socks5.go b/server/proxy/socks5.go index a195693..d79be72 100755 --- a/server/proxy/socks5.go +++ b/server/proxy/socks5.go @@ -199,7 +199,7 @@ func (s *Sock5ModeServer) handleConn(c net.Conn) { c.Close() return } - if (s.task.Client.Cnf.U != "" && s.task.Client.Cnf.P != "") || (s.task.MultiUser != nil && len(s.task.MultiUser.UserMap) > 0) { + if (s.task.Client.Cnf.U != "" && s.task.Client.Cnf.P != "") || (s.task.MultiAccount != nil && len(s.task.MultiAccount.AccountMap) > 0) { buf[1] = UserPassAuth c.Write(buf) if err := s.Auth(c); err != nil { @@ -238,11 +238,11 @@ func (s *Sock5ModeServer) Auth(c net.Conn) error { } var U, P string - if s.task.MultiUser != nil { + if s.task.MultiAccount != nil { // enable multi user auth U = string(user) var ok bool - P, ok = s.task.MultiUser.UserMap[U] + P, ok = s.task.MultiAccount.AccountMap[U] if !ok { return errors.New("验证不通过") } From d4a6560d9a1463c143e455be734e3755f5dc29b2 Mon Sep 17 00:00:00 2001 From: zhangwei Date: Sun, 15 Sep 2019 20:35:21 +0800 Subject: [PATCH 07/34] new feature multi user auth with socks5 --- README.md | 2 ++ conf/multi_account.conf | 2 ++ conf/multiuser.conf | 0 conf/npc.conf | 1 + 4 files changed, 5 insertions(+) create mode 100644 conf/multi_account.conf delete mode 100644 conf/multiuser.conf diff --git a/README.md b/README.md index 1dd7f77..f46d48a 100644 --- a/README.md +++ b/README.md @@ -538,11 +538,13 @@ vkey=123 [socks5] mode=socks5 server_port=9004 +multi_account=multi_account.conf ``` 项 | 含义 ---|--- mode | socks5 server_port | 在服务端的代理端口 +multi_account | socks5多账号配置文件（可选） ##### 私密代理模式 ```ini diff --git a/conf/multi_account.conf b/conf/multi_account.conf new file mode 100644 index 0000000..e3cd792 --- /dev/null +++ b/conf/multi_account.conf @@ -0,0 +1,2 @@ +# key -> user | value -> pwd +npc=npc.pwd \ No newline at end of file diff --git a/conf/multiuser.conf b/conf/multiuser.conf deleted file mode 100644 index e69de29..0000000 diff --git a/conf/npc.conf b/conf/npc.conf index d4a31a8..03a684e 100644 --- a/conf/npc.conf +++ b/conf/npc.conf @@ -40,6 +40,7 @@ server_port=10000 [socks5] mode=socks5 server_port=19009 +multi_account=multi_account.conf # 多账户配置文件,配置及代表启动多账户 [file] mode=file From af8d4a8c12fccf95329e85fda92ce391cb753656 Mon Sep 17 00:00:00 2001 From: he2 Date: Thu, 19 Sep 2019 09:17:32 +0800 Subject: [PATCH 08/34] fix: close health check tcp connection --- client/health.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/client/health.go b/client/health.go index ff2338f..12e35d9 100644 --- a/client/health.go +++ b/client/health.go @@ -2,15 +2,16 @@ package client import ( "container/heap" + "net" + "net/http" + "strings" + "time" + "github.com/cnlh/nps/lib/conn" "github.com/cnlh/nps/lib/file" "github.com/cnlh/nps/lib/sheap" "github.com/cnlh/nps/vender/github.com/astaxie/beego/logs" "github.com/pkg/errors" - "net" - "net/http" - "strings" - "time" ) var isStart bool @@ -70,7 +71,11 @@ func check(t *file.Health) { var rs *http.Response for _, v := range arr { if t.HealthCheckType == "tcp" { - _, err = net.DialTimeout("tcp", v, time.Duration(t.HealthCheckTimeout)*time.Second); + var c net.Conn + c, err = net.DialTimeout("tcp", v, time.Duration(t.HealthCheckTimeout)*time.Second) + if err == nil { + c.Close() + } } else { client := &http.Client{} client.Timeout = time.Duration(t.HealthCheckTimeout) * time.Second From ee50a67f030e96c85a9eed33b9f95962b65e0b2d Mon Sep 17 00:00:00 2001 From: zhangwei Date: Thu, 19 Sep 2019 21:16:47 +0800 Subject: [PATCH 09/34] new feature multi user auth with socks5 --- conf/npc.conf | 2 +- go.mod | 9 ++++++--- go.sum | 26 +++++++++++++++++--------- npc.conf | 21 --------------------- 4 files changed, 24 insertions(+), 34 deletions(-) delete mode 100644 npc.conf diff --git a/conf/npc.conf b/conf/npc.conf index 03a684e..b3dccdb 100644 --- a/conf/npc.conf +++ b/conf/npc.conf @@ -40,7 +40,7 @@ server_port=10000 [socks5] mode=socks5 server_port=19009 -multi_account=multi_account.conf # 多账户配置文件,配置及代表启动多账户 +multi_account=multi_account.conf [file] mode=file diff --git a/go.mod b/go.mod index e8111a1..1f6b753 100644 --- a/go.mod +++ b/go.mod @@ -5,20 +5,23 @@ go 1.12 require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/astaxie/beego v1.12.0 + github.com/belogik/goes v0.0.0-20151229125003-e54d722c3aff // indirect github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d github.com/go-ole/go-ole v1.2.4 // indirect github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db github.com/klauspost/cpuid v1.2.1 // indirect - github.com/klauspost/reedsolomon v1.9.2 // indirect + github.com/klauspost/reedsolomon v1.9.2 + github.com/onsi/gomega v1.5.0 // indirect github.com/pkg/errors v0.8.0 github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 // indirect github.com/shirou/gopsutil v2.18.12+incompatible github.com/stretchr/testify v1.3.0 // indirect github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161 // indirect - github.com/templexxx/xor v0.0.0-20181023030647-4e92f724b73b // indirect - github.com/tjfoc/gmsm v1.0.1 // indirect + github.com/templexxx/xor v0.0.0-20181023030647-4e92f724b73b + github.com/tjfoc/gmsm v1.0.1 github.com/xtaci/kcp-go v5.4.4+incompatible github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae // indirect + golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa // indirect ) diff --git a/go.sum b/go.sum index a0396f9..d57b283 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,11 @@ github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OwnLocal/goes v1.0.0/go.mod h1:8rIFjBGTue3lCU0wplczcUgt9Gxgrkkrw7etMIcn8TM= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/astaxie/beego v1.12.0 h1:MRhVoeeye5N+Flul5PoVfD9CslfdoH+xqC/xvSQ5u2Y= +github.com/astaxie/beego v1.12.0/go.mod h1:fysx+LZNZKnvh4GED/xND7jWtjCR6HzydR2Hh2Im57o= github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd/go.mod h1:1b+Y/CofkYwXMUU0OhQqGvsY2Bvgr4j6jfT699wyZKQ= github.com/beego/x2j v0.0.0-20131220205130-a0352aadc542/go.mod h1:kSeGC/p1AbBiEp5kat81+DSQrZenVBZXklMLaELspWU= +github.com/belogik/goes v0.0.0-20151229125003-e54d722c3aff/go.mod h1:PhH1ZhyCzHKt4uAasyx+ljRCgoezetRNf59CUtwUkqY= github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= github.com/casbin/casbin v1.7.0/go.mod h1:c67qKN6Oum3UF5Q1+BByfFxkwKvhwW57ITjqwtzR1KE= github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d h1:As4937T5NVbJ/DmZT9z33pyLEprMd6CUSfhbmMY57Io= @@ -13,31 +15,34 @@ github.com/couchbase/go-couchbase v0.0.0-20181122212707-3e9b6e1258bb/go.mod h1:T github.com/couchbase/gomemcached v0.0.0-20181122193126-5125a94a666c/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c= github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs= github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGiisLwp9rITslkFNpZD5rz43tf41QFkTWY= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elazarl/go-bindata-assetfs v1.0.0 h1:G/bYguwHIzWq9ZoyUQqrjTmJbbYn3j3CKKpKinvZLFk= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/exfly/beego v1.12.0-export-init h1:VQNYKdXhAwZGUaFmQv8Aj921O3rQJZRIF8xeGrhsjrI= github.com/exfly/beego v1.12.0-export-init/go.mod h1:fysx+LZNZKnvh4GED/xND7jWtjCR6HzydR2Hh2Im57o= -github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= +github.com/exfly/beego v1.12.0 h1:OXwIwngaAx35Mga+jLiZmArusBxj8/H0jYXzGDAdwOg= +github.com/exfly/beego v1.12.0/go.mod h1:fysx+LZNZKnvh4GED/xND7jWtjCR6HzydR2Hh2Im57o= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/reedsolomon v1.9.2 h1:E9CMS2Pqbv+C7tsrYad4YC9MfhnMVWhMRsTi7U0UB18= github.com/klauspost/reedsolomon v1.9.2/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 h1:X+yvsM2yrEktyI+b2qND5gpH8YhURn0k8OCaeRnkINo= github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644/go.mod h1:nkxAfR/5quYxwPZhyDxgasBMnRtBZd0FCEpawpjMUFg= @@ -48,7 +53,6 @@ github.com/siddontang/ledisdb v0.0.0-20181029004158-becf5f38d373/go.mod h1:mF1Dp github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d/go.mod h1:AMEsy7v5z92TR1JKMkLLoaOQk++LVnOKL3ScbJ8GNGA= github.com/ssdb/gossdb v0.0.0-20180723034631-88f6b59b84ec/go.mod h1:QBvMkMya+gXctz3kmljlUCu/yB3GZ6oee+dUozsezQE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161 h1:89CEmDvlq/F7SJEOqkIdNDGJXrQIhuIx9D2DBXjavSU= @@ -60,15 +64,19 @@ github.com/tjfoc/gmsm v1.0.1/go.mod h1:XxO4hdhhrzAd+G4CjDqaOkd0hUzmtPR/d3EiBBMn/ github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc= github.com/xtaci/kcp-go v5.4.4+incompatible h1:QIJ0a0Q0N1G20yLHL2+fpdzyy2v/Cb3PI+xiwx/KK9c= github.com/xtaci/kcp-go v5.4.4+incompatible/go.mod h1:bN6vIwHQbfHaHtFpEssmWsN45a+AZwO7eyRCmEIbtvE= -github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae h1:J0GxkO96kL4WF+AIT3M4mfUVinOCPgf2uUWYFUzN0sM= github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE= golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85 h1:et7+NAX3lLIk5qUCTA9QelBjGE/NkhzYw/mhnr0s7nI= golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a h1:gOpx8G595UYyvj8UK4+OFyY4rx037g3fmfhe5SasG3U= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa h1:KIDDMLT1O0Nr7TSxp8xM5tJcdn8tgyAONntO829og1M= golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= \ No newline at end of file diff --git a/npc.conf b/npc.conf deleted file mode 100644 index 63a8a37..0000000 --- a/npc.conf +++ /dev/null @@ -1,21 +0,0 @@ -[common] -server_addr=proxy.njcp.info:55000 -conn_type=tcp -vkey=%EEKn6q386JmCQC^ -auto_reconnection=true -max_conn=1000 -flow_limit=1000 -rate_limit=1000 -basic_username=bobanNanJing -basic_password=N^#xf407W%ZfOg$u -web_username=12 -web_password=12 -crypt=true -compress=true -username=bobanNanJing -password=0Ho@^#U&y571%cK* - - -[socks5] -mode=socks5 -server_port=19009 \ No newline at end of file From a28d7319d8b2e8746cf5b094b858a52c3a6590fb Mon Sep 17 00:00:00 2001 From: zhangwei Date: Mon, 23 Sep 2019 16:33:42 +0800 Subject: [PATCH 10/34] optimizing readme for socks5 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f46d48a..d1b95af 100644 --- a/README.md +++ b/README.md @@ -544,7 +544,7 @@ multi_account=multi_account.conf ---|--- mode | socks5 server_port | 在服务端的代理端口 -multi_account | socks5多账号配置文件（可选） +multi_account | socks5多账号配置文件（可选),配置后使用basic_username和basic_password无法通过认证 ##### 私密代理模式 ```ini From b3ed822c72c9ec31f873151edd4d16333046651a Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Mon, 7 Oct 2019 23:04:54 +0800 Subject: [PATCH 11/34] change slide window design --- lib/common/const.go | 4 +- lib/common/netpackager.go | 37 +- lib/mux/conn.go | 703 ++++++++++++++++++++++---------------- lib/mux/mux.go | 118 ++++--- lib/mux/queue.go | 148 ++++++-- 5 files changed, 623 insertions(+), 387 deletions(-) diff --git a/lib/common/const.go b/lib/common/const.go index d77f16b..95364a2 100644 --- a/lib/common/const.go +++ b/lib/common/const.go @@ -42,9 +42,11 @@ const ( MUX_NEW_CONN_OK MUX_NEW_CONN_Fail MUX_NEW_MSG + MUX_NEW_MSG_PART MUX_MSG_SEND_OK MUX_NEW_CONN MUX_CONN_CLOSE MUX_PING_RETURN - MUX_PING int32 = -1 + MUX_PING int32 = -1 + MAXIMUM_SEGMENT_SIZE = 4096 - 16 - 32 - 32 - 8 ) diff --git a/lib/common/netpackager.go b/lib/common/netpackager.go index 1129940..ec2cb69 100644 --- a/lib/common/netpackager.go +++ b/lib/common/netpackager.go @@ -15,7 +15,7 @@ type NetPackager interface { } type BasePackager struct { - Length uint32 + Length uint16 Content []byte } @@ -101,7 +101,7 @@ func (Self *BasePackager) Unmarshal(content interface{}) (err error) { } func (Self *BasePackager) setLength() { - Self.Length = uint32(len(Self.Content)) + Self.Length = uint16(len(Self.Content)) return } @@ -147,25 +147,32 @@ func (Self *ConnPackager) UnPack(reader io.Reader) (err error) { } type MuxPackager struct { - Flag uint8 - Id int32 - Window uint16 + Flag uint8 + Id int32 + Window uint32 + ReadLength uint32 BasePackager } func (Self *MuxPackager) NewPac(flag uint8, id int32, content ...interface{}) (err error) { Self.Flag = flag Self.Id = id - if flag == MUX_NEW_MSG { + if flag == MUX_NEW_MSG || flag == MUX_NEW_MSG_PART || flag == MUX_PING_FLAG { err = Self.BasePackager.NewPac(content...) } if flag == MUX_MSG_SEND_OK { // MUX_MSG_SEND_OK only allows one data switch content[0].(type) { case int: - Self.Window = uint16(content[0].(int)) - case uint16: - Self.Window = content[0].(uint16) + Self.Window = uint32(content[0].(int)) + case uint32: + Self.Window = content[0].(uint32) + } + switch content[1].(type) { + case int: + Self.ReadLength = uint32(content[1].(int)) + case uint32: + Self.ReadLength = content[1].(uint32) } } return @@ -180,11 +187,15 @@ func (Self *MuxPackager) Pack(writer io.Writer) (err error) { if err != nil { return } - if Self.Flag == MUX_NEW_MSG { + if Self.Flag == MUX_NEW_MSG || Self.Flag == MUX_NEW_MSG_PART || Self.Flag == MUX_PING_FLAG { err = Self.BasePackager.Pack(writer) } if Self.Flag == MUX_MSG_SEND_OK { err = binary.Write(writer, binary.LittleEndian, Self.Window) + if err != nil { + return + } + err = binary.Write(writer, binary.LittleEndian, Self.ReadLength) } return } @@ -199,11 +210,15 @@ func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { if err != nil { return } - if Self.Flag == MUX_NEW_MSG { + if Self.Flag == MUX_NEW_MSG || Self.Flag == MUX_NEW_MSG_PART || Self.Flag == MUX_PING_FLAG { err = Self.BasePackager.UnPack(reader) } if Self.Flag == MUX_MSG_SEND_OK { err = binary.Read(reader, binary.LittleEndian, &Self.Window) + if err != nil { + return + } + err = binary.Read(reader, binary.LittleEndian, &Self.ReadLength) } return } diff --git a/lib/mux/conn.go b/lib/mux/conn.go index bf9e0d6..f4d5396 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -2,7 +2,9 @@ package mux import ( "errors" + "github.com/astaxie/beego/logs" "io" + "math" "net" "sync" "time" @@ -15,16 +17,11 @@ type conn struct { getStatusCh chan struct{} connStatusOkCh chan struct{} connStatusFailCh chan struct{} - readTimeOut time.Time - writeTimeOut time.Time connId int32 isClose bool closeFlag bool // close conn flag - receiveWindow *window - sendWindow *window - readCh waitingCh - writeCh waitingCh - mux *Mux + receiveWindow *ReceiveWindow + sendWindow *SendWindow once sync.Once } @@ -34,15 +31,12 @@ func NewConn(connId int32, mux *Mux) *conn { connStatusOkCh: make(chan struct{}), connStatusFailCh: make(chan struct{}), connId: connId, - receiveWindow: new(window), - sendWindow: new(window), - mux: mux, + receiveWindow: new(ReceiveWindow), + sendWindow: new(SendWindow), once: sync.Once{}, } - c.receiveWindow.NewReceive() - c.sendWindow.NewSend() - c.readCh.new() - c.writeCh.new() + c.receiveWindow.New(mux) + c.sendWindow.New(mux) return c } @@ -50,39 +44,14 @@ func (s *conn) Read(buf []byte) (n int, err error) { if s.isClose || buf == nil { return 0, errors.New("the conn has closed") } + if len(buf) == 0 { + return 0, nil + } // waiting for takeout from receive window finish or timeout - go s.readWindow(buf, s.readCh.nCh, s.readCh.errCh) - if t := s.readTimeOut.Sub(time.Now()); t > 0 { - timer := time.NewTimer(t) - defer timer.Stop() - select { - case <-timer.C: - return 0, errors.New("read timeout") - case n = <-s.readCh.nCh: - err = <-s.readCh.errCh - } - } else { - n = <-s.readCh.nCh - err = <-s.readCh.errCh - } + n, err = s.receiveWindow.Read(buf, s.connId) return } -func (s *conn) readWindow(buf []byte, nCh chan int, errCh chan error) { - n, err := s.receiveWindow.Read(buf) - if s.receiveWindow.WindowFull { - if s.receiveWindow.Size() > 0 { - // window.Read may be invoked before window.Write, and WindowFull flag change to true - // so make sure that receiveWindow is free some space - s.receiveWindow.WindowFull = false - s.mux.sendInfo(common.MUX_MSG_SEND_OK, s.connId, s.receiveWindow.Size()) - // acknowledge other side, have empty some receive window space - } - } - nCh <- n - errCh <- err -} - func (s *conn) Write(buf []byte) (n int, err error) { if s.isClose { return 0, errors.New("the conn has closed") @@ -91,45 +60,13 @@ func (s *conn) Write(buf []byte) (n int, err error) { //s.Close() return 0, errors.New("io: write on closed conn") } - s.sendWindow.SetSendBuf(buf) // set the buf to send window - go s.write(s.writeCh.nCh, s.writeCh.errCh) - // waiting for send to other side or timeout - if t := s.writeTimeOut.Sub(time.Now()); t > 0 { - timer := time.NewTimer(t) - defer timer.Stop() - select { - case <-timer.C: - return 0, errors.New("write timeout") - case n = <-s.writeCh.nCh: - err = <-s.writeCh.errCh - } - } else { - n = <-s.writeCh.nCh - err = <-s.writeCh.errCh + if len(buf) == 0 { + return 0, nil } + //logs.Warn("write buf", len(buf)) + n, err = s.sendWindow.WriteFull(buf, s.connId) return } -func (s *conn) write(nCh chan int, errCh chan error) { - var n int - var err error - for { - buf, err := s.sendWindow.WriteTo() - // get the usable window size buf from send window - if buf == nil && err == io.EOF { - // send window is drain, break the loop - err = nil - break - } - if err != nil { - break - } - n += len(buf) - s.mux.sendInfo(common.MUX_NEW_MSG, s.connId, buf) - // send to other side, not send nil data to other side - } - nCh <- n - errCh <- err -} func (s *conn) Close() (err error) { s.once.Do(s.closeProcess) @@ -138,11 +75,11 @@ func (s *conn) Close() (err error) { func (s *conn) closeProcess() { s.isClose = true - s.mux.connMap.Delete(s.connId) - if !s.mux.IsClose { + s.receiveWindow.mux.connMap.Delete(s.connId) + if !s.receiveWindow.mux.IsClose { // if server or user close the conn while reading, will get a io.EOF // and this Close method will be invoke, send this signal to close other side - s.mux.sendInfo(common.MUX_CONN_CLOSE, s.connId, nil) + s.receiveWindow.mux.sendInfo(common.MUX_CONN_CLOSE, s.connId, nil) } s.sendWindow.CloseWindow() s.receiveWindow.CloseWindow() @@ -150,276 +87,440 @@ func (s *conn) closeProcess() { } func (s *conn) LocalAddr() net.Addr { - return s.mux.conn.LocalAddr() + return s.receiveWindow.mux.conn.LocalAddr() } func (s *conn) RemoteAddr() net.Addr { - return s.mux.conn.RemoteAddr() + return s.receiveWindow.mux.conn.RemoteAddr() } func (s *conn) SetDeadline(t time.Time) error { - s.readTimeOut = t - s.writeTimeOut = t + _ = s.SetReadDeadline(t) + _ = s.SetWriteDeadline(t) return nil } func (s *conn) SetReadDeadline(t time.Time) error { - s.readTimeOut = t + s.receiveWindow.SetTimeOut(t) return nil } func (s *conn) SetWriteDeadline(t time.Time) error { - s.writeTimeOut = t + s.sendWindow.SetTimeOut(t) return nil } type window struct { - windowBuff []byte - off uint16 - readOp chan struct{} - readWait bool - WindowFull bool - usableReceiveWindow chan uint16 - WriteWg sync.WaitGroup - closeOp bool - closeOpCh chan struct{} - WriteEndOp chan struct{} - mutex sync.Mutex + off uint32 + maxSize uint32 + closeOp bool + closeOpCh chan struct{} + mux *Mux } -func (Self *window) NewReceive() { +func (Self *window) New() { + Self.closeOpCh = make(chan struct{}, 2) +} + +func (Self *window) CloseWindow() { + if !Self.closeOp { + Self.closeOp = true + Self.closeOpCh <- struct{}{} + Self.closeOpCh <- struct{}{} + } +} + +type ReceiveWindow struct { + bufQueue FIFOQueue + element *ListElement + readLength uint32 + readOp chan struct{} + readWait bool + windowFull bool + count int8 + bw *bandwidth + once sync.Once + window +} + +func (Self *ReceiveWindow) New(mux *Mux) { // initial a window for receive - Self.windowBuff = common.WindowBuff.Get() Self.readOp = make(chan struct{}) - Self.WriteEndOp = make(chan struct{}) - Self.closeOpCh = make(chan struct{}, 3) + Self.bufQueue.New() + Self.bw = new(bandwidth) + Self.element = new(ListElement) + Self.maxSize = 8192 + Self.mux = mux + Self.window.New() } -func (Self *window) NewSend() { - // initial a window for send - Self.usableReceiveWindow = make(chan uint16) - Self.closeOpCh = make(chan struct{}, 3) +func (Self *ReceiveWindow) RemainingSize() (n uint32) { + // receive window remaining + if Self.maxSize >= Self.bufQueue.Len() { + n = Self.maxSize - Self.bufQueue.Len() + } + // if maxSize is small than bufQueue length, return 0 + return } -func (Self *window) SetSendBuf(buf []byte) { +func (Self *ReceiveWindow) ReadSize() (n uint32) { + // acknowledge the size already read + Self.bufQueue.mutex.Lock() + n = Self.readLength + Self.readLength = 0 + Self.bufQueue.mutex.Unlock() + Self.count += 1 + return +} + +func (Self *ReceiveWindow) CalcSize() { + // calculating maximum receive window size + if Self.count == 0 { + logs.Warn("ping, bw", Self.mux.latency, Self.bw.Get()) + n := uint32(2 * Self.mux.latency * Self.bw.Get()) + if n < 8192 { + n = 8192 + } + if n < Self.bufQueue.Len() { + n = Self.bufQueue.Len() + } + // set the minimal size + logs.Warn("n", n) + Self.maxSize = n + Self.count = -5 + } +} + +func (Self *ReceiveWindow) Write(buf []byte, l uint16, part bool, id int32) (err error) { + if Self.closeOp { + return errors.New("conn.receiveWindow: write on closed window") + } + element := ListElement{} + err = element.New(buf, l, part) + //logs.Warn("push the buf", len(buf), l, (&element).l) + if err != nil { + return + } + Self.bufQueue.Push(&element) // must push data before allow read + //logs.Warn("read session calc size ", Self.maxSize) + // calculating the receive window size + Self.CalcSize() + logs.Warn("read session calc size finish", Self.maxSize) + if Self.RemainingSize() == 0 { + Self.windowFull = true + //logs.Warn("window full true", Self.windowFull) + } + Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.ReadSize()) + return nil +} + +func (Self *ReceiveWindow) Read(p []byte, id int32) (n int, err error) { + if Self.closeOp { + return 0, io.EOF // receive close signal, returns eof + } + pOff := 0 + l := 0 + //logs.Warn("receive window read off, element.l", Self.off, Self.element.l) +copyData: + Self.bw.StartRead() + if Self.off == uint32(Self.element.l) { + // on the first Read method invoked, Self.off and Self.element.l + // both zero value + Self.element, err = Self.bufQueue.Pop() + // if the queue is empty, Pop method will wait until one element push + // into the queue successful, or timeout. + // timer start on timeout parameter is set up , + // reset to 60s if timeout and data still available + Self.off = 0 + if err != nil { + return // queue receive stop or time out, break the loop and return + } + //logs.Warn("pop element", Self.element.l, Self.element.part) + } + l = copy(p[pOff:], Self.element.buf[Self.off:]) + Self.bw.SetCopySize(l) + pOff += l + Self.off += uint32(l) + Self.bufQueue.mutex.Lock() + Self.readLength += uint32(l) + //logs.Warn("window read length buf len", Self.readLength, Self.bufQueue.Len()) + Self.bufQueue.mutex.Unlock() + n += l + l = 0 + Self.bw.EndRead() + Self.sendStatus(id) + if pOff < len(p) && Self.element.part { + // element is a part of the segments, trying to fill up buf p + goto copyData + } + return // buf p is full or all of segments in buf, return +} + +func (Self *ReceiveWindow) sendStatus(id int32) { + if Self.windowFull || Self.bufQueue.Len() == 0 { + // window is full before read or empty now + Self.windowFull = false + Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.ReadSize()) + // acknowledge other side, have empty some receive window space + //} + } +} + +func (Self *ReceiveWindow) SetTimeOut(t time.Time) { + // waiting for FIFO queue Pop method + Self.bufQueue.SetTimeOut(t) +} + +func (Self *ReceiveWindow) Stop() { + // queue has no more data to push, so unblock pop method + Self.once.Do(Self.bufQueue.Stop) +} + +func (Self *ReceiveWindow) CloseWindow() { + Self.window.CloseWindow() + Self.Stop() +} + +type SendWindow struct { + buf []byte + sentLength uint32 + setSizeCh chan struct{} + setSizeWait bool + unSlide uint32 + timeout time.Time + window + mutex sync.Mutex +} + +func (Self *SendWindow) New(mux *Mux) { + Self.setSizeCh = make(chan struct{}) + Self.maxSize = 4096 + Self.mux = mux + Self.window.New() +} + +func (Self *SendWindow) SetSendBuf(buf []byte) { // send window buff from conn write method, set it to send window Self.mutex.Lock() - Self.windowBuff = buf + Self.buf = buf Self.off = 0 Self.mutex.Unlock() } -func (Self *window) fullSlide() { - // slide by allocate - newBuf := common.WindowBuff.Get() - Self.liteSlide() - n := copy(newBuf[:Self.len()], Self.windowBuff) - common.WindowBuff.Put(Self.windowBuff) - Self.windowBuff = newBuf[:n] +func (Self *SendWindow) RemainingSize() (n uint32) { + if Self.maxSize >= Self.sentLength { + n = Self.maxSize - Self.sentLength + } return } -func (Self *window) liteSlide() { - // slide by re slice - Self.windowBuff = Self.windowBuff[Self.off:] - Self.off = 0 - return -} - -func (Self *window) Size() (n int) { - // receive Window remaining - n = common.PoolSizeWindow - Self.len() - return -} - -func (Self *window) len() (n int) { - n = len(Self.windowBuff[Self.off:]) - return -} - -func (Self *window) cap() (n int) { - n = cap(Self.windowBuff[Self.off:]) - return -} - -func (Self *window) grow(n int) { - Self.windowBuff = Self.windowBuff[:Self.len()+n] -} - -func (Self *window) Write(p []byte) (n int, err error) { - if Self.closeOp { - return 0, errors.New("conn.receiveWindow: write on closed window") - } - if len(p) > Self.Size() { - return 0, errors.New("conn.receiveWindow: write too large") - } - Self.mutex.Lock() - // slide the offset - if len(p) > Self.cap()-Self.len() { - // not enough space, need to allocate - Self.fullSlide() - } else { - // have enough space, re slice - Self.liteSlide() - } - length := Self.len() // length before grow - Self.grow(len(p)) // grow for copy - n = copy(Self.windowBuff[length:], p) // must copy data before allow Read - if Self.readWait { - // if there condition is length == 0 and - // Read method just take away all the windowBuff, - // this method will block until windowBuff is empty again - - // allow continue read - defer Self.allowRead() - } - Self.mutex.Unlock() - return n, nil -} - -func (Self *window) allowRead() (closed bool) { - if Self.closeOp { - close(Self.readOp) - return true - } - Self.mutex.Lock() - Self.readWait = false - Self.mutex.Unlock() - select { - case <-Self.closeOpCh: - close(Self.readOp) - return true - case Self.readOp <- struct{}{}: - return false - } -} - -func (Self *window) Read(p []byte) (n int, err error) { - if Self.closeOp { - return 0, io.EOF // Write method receive close signal, returns eof - } - Self.mutex.Lock() - length := Self.len() // protect the length data, it invokes - // before Write lock and after Write unlock - if length == 0 { - // window is empty, waiting for Write method send a success readOp signal - // or get timeout or close - Self.readWait = true - Self.mutex.Unlock() - ticker := time.NewTicker(2 * time.Minute) - defer ticker.Stop() - select { - case _, ok := <-Self.readOp: - if !ok { - return 0, errors.New("conn.receiveWindow: window closed") - } - case <-Self.WriteEndOp: - return 0, io.EOF // receive eof signal, returns eof - case <-ticker.C: - return 0, errors.New("conn.receiveWindow: read time out") - case <-Self.closeOpCh: - close(Self.readOp) - return 0, io.EOF // receive close signal, returns eof - } - } else { - Self.mutex.Unlock() - } - minCopy := 512 - for { - Self.mutex.Lock() - if len(p) == n || Self.len() == 0 { - Self.mutex.Unlock() - break - } - if n+minCopy > len(p) { - minCopy = len(p) - n - } - i := copy(p[n:n+minCopy], Self.windowBuff[Self.off:]) - Self.off += uint16(i) - n += i - Self.mutex.Unlock() - } - p = p[:n] - return -} - -func (Self *window) WriteTo() (p []byte, err error) { - if Self.closeOp { - return nil, errors.New("conn.writeWindow: window closed") - } - if Self.len() == 0 { - return nil, io.EOF - // send window buff is drain, return eof and get another one - } - var windowSize uint16 - var ok bool -waiting: - ticker := time.NewTicker(2 * time.Minute) - defer ticker.Stop() - // waiting for receive usable window size, or timeout - select { - case windowSize, ok = <-Self.usableReceiveWindow: - if !ok { - return nil, errors.New("conn.writeWindow: window closed") - } - case <-ticker.C: - return nil, errors.New("conn.writeWindow: write to time out") - case <-Self.closeOpCh: - return nil, errors.New("conn.writeWindow: window closed") - } - if windowSize == 0 { - goto waiting // waiting for another usable window size - } - Self.mutex.Lock() - if windowSize > uint16(Self.len()) { - // usable window size is bigger than window buff size, send the full buff - windowSize = uint16(Self.len()) - } - p = Self.windowBuff[Self.off : windowSize+Self.off] - Self.off += windowSize - Self.mutex.Unlock() - return -} - -func (Self *window) SetAllowSize(value uint16) (closed bool) { +func (Self *SendWindow) SetSize(windowSize, readLength uint32) (closed bool) { defer func() { if recover() != nil { closed = true } }() if Self.closeOp { - close(Self.usableReceiveWindow) + close(Self.setSizeCh) return true } - select { - case Self.usableReceiveWindow <- value: - return false - case <-Self.closeOpCh: - close(Self.usableReceiveWindow) - return true + if readLength == 0 && Self.maxSize == windowSize { + logs.Warn("waiting for another window size") + return false // waiting for receive another usable window size } + logs.Warn("set send window size to ", windowSize, readLength) + Self.mutex.Lock() + Self.slide(windowSize, readLength) + if Self.setSizeWait { + // send window into the wait status, need notice the channel + //logs.Warn("send window remaining size is 0 , wait") + if Self.RemainingSize() == 0 { + //logs.Warn("waiting for another window size after slide") + // keep the wait status + Self.mutex.Unlock() + return false + } + Self.setSizeWait = false + Self.mutex.Unlock() + //logs.Warn("send window remaining size is 0 starting wait") + select { + case Self.setSizeCh <- struct{}{}: + //logs.Warn("send window remaining size is 0 finish") + return false + case <-Self.closeOpCh: + close(Self.setSizeCh) + return true + } + } + // send window not into the wait status, so just do slide + Self.mutex.Unlock() + return false } -func (Self *window) CloseWindow() { - Self.closeOp = true - Self.closeOpCh <- struct{}{} - Self.closeOpCh <- struct{}{} - Self.closeOpCh <- struct{}{} - close(Self.closeOpCh) +func (Self *SendWindow) slide(windowSize, readLength uint32) { + Self.sentLength -= readLength + Self.maxSize = windowSize +} + +func (Self *SendWindow) WriteTo() (p []byte, part bool, err error) { + // returns buf segments, return only one segments, need a loop outside + // until err = io.EOF + if Self.closeOp { + return nil, false, errors.New("conn.writeWindow: window closed") + } + if Self.off == uint32(len(Self.buf)) { + return nil, false, io.EOF + // send window buff is drain, return eof and get another one + } + Self.mutex.Lock() + if Self.RemainingSize() == 0 { + Self.setSizeWait = true + Self.mutex.Unlock() + // into the wait status + err = Self.waitReceiveWindow() + if err != nil { + return nil, false, err + } + } else { + Self.mutex.Unlock() + } + Self.mutex.Lock() + var sendSize uint32 + if len(Self.buf[Self.off:]) > common.MAXIMUM_SEGMENT_SIZE { + sendSize = common.MAXIMUM_SEGMENT_SIZE + part = true + } else { + sendSize = uint32(len(Self.buf[Self.off:])) + part = false + } + if Self.RemainingSize() < sendSize { + // usable window size is small than + // window MAXIMUM_SEGMENT_SIZE or send buf left + sendSize = Self.RemainingSize() + part = true + } + //logs.Warn("send size", sendSize) + p = Self.buf[Self.off : sendSize+Self.off] + Self.off += sendSize + Self.sentLength += sendSize + Self.mutex.Unlock() return } -type waitingCh struct { - nCh chan int - errCh chan error +func (Self *SendWindow) waitReceiveWindow() (err error) { + t := Self.timeout.Sub(time.Now()) + if t < 0 { + t = time.Minute + } + timer := time.NewTimer(t) + defer timer.Stop() + // waiting for receive usable window size, or timeout + select { + case _, ok := <-Self.setSizeCh: + if !ok { + return errors.New("conn.writeWindow: window closed") + } + return nil + case <-timer.C: + return errors.New("conn.writeWindow: write to time out") + case <-Self.closeOpCh: + return errors.New("conn.writeWindow: window closed") + } } -func (Self *waitingCh) new() { - Self.nCh = make(chan int) - Self.errCh = make(chan error) +func (Self *SendWindow) WriteFull(buf []byte, id int32) (n int, err error) { + Self.SetSendBuf(buf) // set the buf to send window + var bufSeg []byte + var part bool + for { + bufSeg, part, err = Self.WriteTo() + //logs.Warn("buf seg", len(bufSeg), part, err) + // get the buf segments from send window + if bufSeg == nil && part == false && err == io.EOF { + // send window is drain, break the loop + err = nil + break + } + if err != nil { + break + } + n += len(bufSeg) + if part { + Self.mux.sendInfo(common.MUX_NEW_MSG_PART, id, bufSeg) + } else { + Self.mux.sendInfo(common.MUX_NEW_MSG, id, bufSeg) + //logs.Warn("buf seg sent", len(bufSeg), part, err) + } + // send to other side, not send nil data to other side + } + //logs.Warn("buf seg write success") + return } -func (Self *waitingCh) close() { - close(Self.nCh) - close(Self.errCh) +func (Self *SendWindow) SetTimeOut(t time.Time) { + // waiting for receive a receive window size + Self.timeout = t +} + +type bandwidth struct { + lastReadStart time.Time + readStart time.Time + readEnd time.Time + bufLength int + lastBufLength int + count int8 + readBW float64 + writeBW float64 +} + +func (Self *bandwidth) StartRead() { + Self.lastReadStart, Self.readStart = Self.readStart, time.Now() +} + +func (Self *bandwidth) EndRead() { + if !Self.lastReadStart.IsZero() { + if Self.count == 0 { + Self.calcWriteBandwidth() + } + } + Self.readEnd = time.Now() + if Self.count == 0 { + Self.calcReadBandwidth() + Self.count = -3 + } + Self.count += 1 +} + +func (Self *bandwidth) SetCopySize(n int) { + // must be invoke between StartRead and EndRead + Self.lastBufLength, Self.bufLength = Self.bufLength, n +} + +func (Self *bandwidth) calcReadBandwidth() { + // Bandwidth between nps and npc + readTime := Self.readEnd.Sub(Self.readStart) + Self.readBW = float64(Self.bufLength) / readTime.Seconds() + //logs.Warn("calc read bw", Self.bufLength, readTime.Seconds()) +} + +func (Self *bandwidth) calcWriteBandwidth() { + // Bandwidth between nps and user, npc and application + //logs.Warn("calc write bw") + writeTime := Self.readEnd.Sub(Self.lastReadStart) + Self.writeBW = float64(Self.lastBufLength) / writeTime.Seconds() +} + +func (Self *bandwidth) Get() (bw float64) { + // The zero value, 0 for numeric types + if Self.writeBW == 0 && Self.readBW == 0 { + logs.Warn("bw both 0") + return 100 + } + if Self.writeBW == 0 && Self.readBW != 0 { + return Self.readBW + } + if Self.readBW == 0 && Self.writeBW != 0 { + return Self.writeBW + } + return math.Min(Self.readBW, Self.writeBW) } diff --git a/lib/mux/mux.go b/lib/mux/mux.go index a662ad0..e6a9e67 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -3,6 +3,7 @@ package mux import ( "bytes" "errors" + "io" "math" "net" "sync" @@ -22,8 +23,10 @@ type Mux struct { closeChan chan struct{} IsClose bool pingOk int + latency float64 + pingCh chan []byte connType string - writeQueue Queue + writeQueue PriorityQueue bufCh chan *bytes.Buffer sync.Mutex } @@ -38,13 +41,15 @@ func NewMux(c net.Conn, connType string) *Mux { IsClose: false, connType: connType, bufCh: make(chan *bytes.Buffer), + pingCh: make(chan []byte), } m.writeQueue.New() //read session by flag - go m.readSession() + m.readSession() //ping - go m.ping() - go m.writeSession() + m.ping() + m.pingReturn() + m.writeSession() return m } @@ -83,10 +88,10 @@ func (s *Mux) Addr() net.Addr { return s.conn.LocalAddr() } -func (s *Mux) sendInfo(flag uint8, id int32, data interface{}) { +func (s *Mux) sendInfo(flag uint8, id int32, data ...interface{}) { var err error pack := common.MuxPack.Get() - err = pack.NewPac(flag, id, data) + err = pack.NewPac(flag, id, data...) if err != nil { common.MuxPack.Put(pack) return @@ -98,11 +103,13 @@ func (s *Mux) sendInfo(flag uint8, id int32, data interface{}) { func (s *Mux) writeSession() { go s.packBuf() go s.writeBuf() - <-s.closeChan } func (s *Mux) packBuf() { for { + if s.IsClose { + break + } pack := s.writeQueue.Pop() buffer := common.BuffPool.Get() err := pack.Pack(buffer) @@ -117,12 +124,14 @@ func (s *Mux) packBuf() { case <-s.closeChan: break } - } } func (s *Mux) writeBuf() { for { + if s.IsClose { + break + } select { case buffer := <-s.bufCh: l := buffer.Len() @@ -141,8 +150,15 @@ func (s *Mux) writeBuf() { func (s *Mux) ping() { go func() { - ticker := time.NewTicker(time.Second * 1) + now, _ := time.Now().MarshalText() + s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) + // send the ping flag and get the latency first + ticker := time.NewTicker(time.Second * 15) for { + if s.IsClose { + ticker.Stop() + break + } select { case <-ticker.C: } @@ -150,7 +166,8 @@ func (s *Mux) ping() { if (math.MaxInt32 - s.id) < 10000 { s.id = 0 } - s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, nil) + now, _ := time.Now().MarshalText() + s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) if s.pingOk > 10 && s.connType == "kcp" { s.Close() break @@ -158,15 +175,32 @@ func (s *Mux) ping() { s.pingOk++ } }() - select { - case <-s.closeChan: - } +} + +func (s *Mux) pingReturn() { + go func() { + var now time.Time + var data []byte + for { + select { + case data = <-s.pingCh: + case <-s.closeChan: + break + } + _ = now.UnmarshalText(data) + s.latency = time.Since(now).Seconds() + s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, nil) + } + }() } func (s *Mux) readSession() { go func() { pack := common.MuxPack.Get() for { + if s.IsClose { + break + } pack = common.MuxPack.Get() if pack.UnPack(s.conn) != nil { break @@ -176,44 +210,25 @@ func (s *Mux) readSession() { case common.MUX_NEW_CONN: //new connection connection := NewConn(pack.Id, s) s.connMap.Set(pack.Id, connection) //it has been set before send ok - go func(connection *conn) { - connection.sendWindow.SetAllowSize(512) // set the initial receive window - }(connection) s.newConnCh <- connection s.sendInfo(common.MUX_NEW_CONN_OK, connection.connId, nil) continue case common.MUX_PING_FLAG: //ping - go s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, nil) + s.pingCh <- pack.Content continue case common.MUX_PING_RETURN: continue } if connection, ok := s.connMap.Get(pack.Id); ok && !connection.isClose { switch pack.Flag { - case common.MUX_NEW_MSG: //new msg from remote connection - //insert wait queue - if connection.isClose { - continue + case common.MUX_NEW_MSG, common.MUX_NEW_MSG_PART: //new msg from remote connection + err := s.newMsg(connection, pack) + if err != nil { + connection.Close() } - connection.receiveWindow.WriteWg.Add(1) - go func(connection *conn, content []byte) { // do not block read session - _, err := connection.receiveWindow.Write(content) - if err != nil { - logs.Warn("mux new msg err close", err) - connection.Close() - } - size := connection.receiveWindow.Size() - if size == 0 { - connection.receiveWindow.WindowFull = true - } - s.sendInfo(common.MUX_MSG_SEND_OK, connection.connId, size) - connection.receiveWindow.WriteWg.Done() - }(connection, pack.Content) continue case common.MUX_NEW_CONN_OK: //connection ok connection.connStatusOkCh <- struct{}{} - go connection.sendWindow.SetAllowSize(512) - // set the initial receive window both side continue case common.MUX_NEW_CONN_Fail: connection.connStatusFailCh <- struct{}{} @@ -222,15 +237,12 @@ func (s *Mux) readSession() { if connection.isClose { continue } - go connection.sendWindow.SetAllowSize(pack.Window) + connection.sendWindow.SetSize(pack.Window, pack.ReadLength) continue case common.MUX_CONN_CLOSE: //close the connection s.connMap.Delete(pack.Id) connection.closeFlag = true - go func(connection *conn) { - connection.receiveWindow.WriteWg.Wait() - connection.receiveWindow.WriteEndOp <- struct{}{} // close signal to receive window - }(connection) + connection.receiveWindow.Stop() // close signal to receive window continue } } else if pack.Flag == common.MUX_CONN_CLOSE { @@ -241,9 +253,24 @@ func (s *Mux) readSession() { common.MuxPack.Put(pack) s.Close() }() - select { - case <-s.closeChan: +} + +func (s *Mux) newMsg(connection *conn, pack *common.MuxPackager) (err error) { + if connection.isClose { + err = io.ErrClosedPipe + return } + //logs.Warn("read session receive new msg", pack.Length) + //go func(connection *conn, pack *common.MuxPackager) { // do not block read session + //insert into queue + if pack.Flag == common.MUX_NEW_MSG_PART { + err = connection.receiveWindow.Write(pack.Content, pack.Length, true, pack.Id) + } + if pack.Flag == common.MUX_NEW_MSG { + err = connection.receiveWindow.Write(pack.Content, pack.Length, false, pack.Id) + } + //logs.Warn("read session write success", pack.Length) + return } func (s *Mux) Close() error { @@ -255,9 +282,6 @@ func (s *Mux) Close() error { s.connMap.Close() s.closeChan <- struct{}{} s.closeChan <- struct{}{} - s.closeChan <- struct{}{} - s.closeChan <- struct{}{} - s.closeChan <- struct{}{} close(s.newConnCh) return s.conn.Close() } diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 081b2c9..5a57151 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -2,25 +2,54 @@ package mux import ( "container/list" + "errors" "github.com/cnlh/nps/lib/common" + "io" "sync" + "time" ) -type Queue struct { - list *list.List +type QueueOp struct { readOp chan struct{} cleanOp chan struct{} popWait bool mutex sync.Mutex } -func (Self *Queue) New() { - Self.list = list.New() +func (Self *QueueOp) New() { Self.readOp = make(chan struct{}) Self.cleanOp = make(chan struct{}, 2) } -func (Self *Queue) Push(packager *common.MuxPackager) { +func (Self *QueueOp) allowPop() (closed bool) { + Self.mutex.Lock() + Self.popWait = false + Self.mutex.Unlock() + select { + case Self.readOp <- struct{}{}: + return false + case <-Self.cleanOp: + return true + } +} + +func (Self *QueueOp) Clean() { + Self.cleanOp <- struct{}{} + Self.cleanOp <- struct{}{} + close(Self.cleanOp) +} + +type PriorityQueue struct { + list *list.List + QueueOp +} + +func (Self *PriorityQueue) New() { + Self.list = list.New() + Self.QueueOp.New() +} + +func (Self *PriorityQueue) Push(packager *common.MuxPackager) { Self.mutex.Lock() if Self.popWait { defer Self.allowPop() @@ -35,28 +64,16 @@ func (Self *Queue) Push(packager *common.MuxPackager) { return } -func (Self *Queue) allowPop() (closed bool) { - Self.mutex.Lock() - Self.popWait = false - Self.mutex.Unlock() - select { - case Self.readOp <- struct{}{}: - return false - case <-Self.cleanOp: - return true - } -} - -func (Self *Queue) insert(packager *common.MuxPackager) { +func (Self *PriorityQueue) insert(packager *common.MuxPackager) { element := Self.list.Back() for { - if element == nil { // Queue dose not have any of msg package with this close package id + if element == nil { // PriorityQueue dose not have any of msg package with this close package id Self.list.PushFront(packager) // insert close package to first break } if element.Value.(*common.MuxPackager).Flag == common.MUX_NEW_MSG && element.Value.(*common.MuxPackager).Id == packager.Id { - Self.list.InsertAfter(packager, element) // Queue has some msg package + Self.list.InsertAfter(packager, element) // PriorityQueue has some msg package // with this close package id, insert close package after last msg package break } @@ -64,7 +81,7 @@ func (Self *Queue) insert(packager *common.MuxPackager) { } } -func (Self *Queue) Pop() (packager *common.MuxPackager) { +func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { Self.mutex.Lock() element := Self.list.Front() if element != nil { @@ -73,7 +90,7 @@ func (Self *Queue) Pop() (packager *common.MuxPackager) { Self.mutex.Unlock() return } - Self.popWait = true // Queue is empty, notice Push method + Self.popWait = true // PriorityQueue is empty, notice Push method Self.mutex.Unlock() select { case <-Self.readOp: @@ -83,13 +100,90 @@ func (Self *Queue) Pop() (packager *common.MuxPackager) { } } -func (Self *Queue) Len() (n int) { +func (Self *PriorityQueue) Len() (n int) { n = Self.list.Len() return } -func (Self *Queue) Clean() { - Self.cleanOp <- struct{}{} - Self.cleanOp <- struct{}{} - close(Self.cleanOp) +type ListElement struct { + buf []byte + l uint16 + part bool +} + +func (Self *ListElement) New(buf []byte, l uint16, part bool) (err error) { + if uint16(len(buf)) != l { + return errors.New("ListElement: buf length not match") + } + Self.buf = buf + Self.l = l + Self.part = part + return nil +} + +type FIFOQueue struct { + list []*ListElement + length uint32 + stopOp chan struct{} + timeout time.Time + QueueOp +} + +func (Self *FIFOQueue) New() { + Self.QueueOp.New() + Self.stopOp = make(chan struct{}, 1) +} + +func (Self *FIFOQueue) Push(element *ListElement) { + Self.mutex.Lock() + if Self.popWait { + defer Self.allowPop() + } + Self.list = append(Self.list, element) + Self.length += uint32(element.l) + Self.mutex.Unlock() + return +} + +func (Self *FIFOQueue) Pop() (element *ListElement, err error) { + Self.mutex.Lock() + if len(Self.list) == 0 { + Self.popWait = true + Self.mutex.Unlock() + t := Self.timeout.Sub(time.Now()) + if t <= 0 { + t = time.Minute + } + timer := time.NewTimer(t) + defer timer.Stop() + select { + case <-Self.readOp: + Self.mutex.Lock() + case <-Self.cleanOp: + return + case <-Self.stopOp: + err = io.EOF + return + case <-timer.C: + err = errors.New("mux.queue: read time out") + return + } + } + element = Self.list[0] + Self.list = Self.list[1:] + Self.length -= uint32(element.l) + Self.mutex.Unlock() + return +} + +func (Self *FIFOQueue) Len() (n uint32) { + return Self.length +} + +func (Self *FIFOQueue) Stop() { + Self.stopOp <- struct{}{} +} + +func (Self *FIFOQueue) SetTimeOut(t time.Time) { + Self.timeout = t } From d9f9dc6acbbe199ff320df2444d375db6835ac69 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Tue, 8 Oct 2019 13:38:42 +0800 Subject: [PATCH 12/34] change ping calculate, fix window size calculate --- lib/common/netpackager.go | 20 ++++++++++---------- lib/mux/conn.go | 15 +++++++-------- lib/mux/mux.go | 14 +++++++++----- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/lib/common/netpackager.go b/lib/common/netpackager.go index ec2cb69..69ce96a 100644 --- a/lib/common/netpackager.go +++ b/lib/common/netpackager.go @@ -157,11 +157,11 @@ type MuxPackager struct { func (Self *MuxPackager) NewPac(flag uint8, id int32, content ...interface{}) (err error) { Self.Flag = flag Self.Id = id - if flag == MUX_NEW_MSG || flag == MUX_NEW_MSG_PART || flag == MUX_PING_FLAG { + switch flag { + case MUX_NEW_MSG, MUX_NEW_MSG_PART, MUX_PING_FLAG, MUX_PING_RETURN: err = Self.BasePackager.NewPac(content...) - } - if flag == MUX_MSG_SEND_OK { - // MUX_MSG_SEND_OK only allows one data + case MUX_MSG_SEND_OK: + // MUX_MSG_SEND_OK contains two data switch content[0].(type) { case int: Self.Window = uint32(content[0].(int)) @@ -187,10 +187,10 @@ func (Self *MuxPackager) Pack(writer io.Writer) (err error) { if err != nil { return } - if Self.Flag == MUX_NEW_MSG || Self.Flag == MUX_NEW_MSG_PART || Self.Flag == MUX_PING_FLAG { + switch Self.Flag { + case MUX_NEW_MSG, MUX_NEW_MSG_PART, MUX_PING_FLAG, MUX_PING_RETURN: err = Self.BasePackager.Pack(writer) - } - if Self.Flag == MUX_MSG_SEND_OK { + case MUX_MSG_SEND_OK: err = binary.Write(writer, binary.LittleEndian, Self.Window) if err != nil { return @@ -210,10 +210,10 @@ func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { if err != nil { return } - if Self.Flag == MUX_NEW_MSG || Self.Flag == MUX_NEW_MSG_PART || Self.Flag == MUX_PING_FLAG { + switch Self.Flag { + case MUX_NEW_MSG, MUX_NEW_MSG_PART, MUX_PING_FLAG, MUX_PING_RETURN: err = Self.BasePackager.UnPack(reader) - } - if Self.Flag == MUX_MSG_SEND_OK { + case MUX_MSG_SEND_OK: err = binary.Read(reader, binary.LittleEndian, &Self.Window) if err != nil { return diff --git a/lib/mux/conn.go b/lib/mux/conn.go index f4d5396..5dd69ea 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -2,7 +2,6 @@ package mux import ( "errors" - "github.com/astaxie/beego/logs" "io" "math" "net" @@ -169,14 +168,13 @@ func (Self *ReceiveWindow) ReadSize() (n uint32) { n = Self.readLength Self.readLength = 0 Self.bufQueue.mutex.Unlock() - Self.count += 1 return } func (Self *ReceiveWindow) CalcSize() { // calculating maximum receive window size if Self.count == 0 { - logs.Warn("ping, bw", Self.mux.latency, Self.bw.Get()) + //logs.Warn("ping, bw", Self.mux.latency, Self.bw.Get()) n := uint32(2 * Self.mux.latency * Self.bw.Get()) if n < 8192 { n = 8192 @@ -185,10 +183,11 @@ func (Self *ReceiveWindow) CalcSize() { n = Self.bufQueue.Len() } // set the minimal size - logs.Warn("n", n) + //logs.Warn("n", n) Self.maxSize = n Self.count = -5 } + Self.count += 1 } func (Self *ReceiveWindow) Write(buf []byte, l uint16, part bool, id int32) (err error) { @@ -205,7 +204,7 @@ func (Self *ReceiveWindow) Write(buf []byte, l uint16, part bool, id int32) (err //logs.Warn("read session calc size ", Self.maxSize) // calculating the receive window size Self.CalcSize() - logs.Warn("read session calc size finish", Self.maxSize) + //logs.Warn("read session calc size finish", Self.maxSize) if Self.RemainingSize() == 0 { Self.windowFull = true //logs.Warn("window full true", Self.windowFull) @@ -325,10 +324,10 @@ func (Self *SendWindow) SetSize(windowSize, readLength uint32) (closed bool) { return true } if readLength == 0 && Self.maxSize == windowSize { - logs.Warn("waiting for another window size") + //logs.Warn("waiting for another window size") return false // waiting for receive another usable window size } - logs.Warn("set send window size to ", windowSize, readLength) + //logs.Warn("set send window size to ", windowSize, readLength) Self.mutex.Lock() Self.slide(windowSize, readLength) if Self.setSizeWait { @@ -513,7 +512,7 @@ func (Self *bandwidth) calcWriteBandwidth() { func (Self *bandwidth) Get() (bw float64) { // The zero value, 0 for numeric types if Self.writeBW == 0 && Self.readBW == 0 { - logs.Warn("bw both 0") + //logs.Warn("bw both 0") return 100 } if Self.writeBW == 0 && Self.readBW != 0 { diff --git a/lib/mux/mux.go b/lib/mux/mux.go index e6a9e67..9023b82 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -150,7 +150,7 @@ func (s *Mux) writeBuf() { func (s *Mux) ping() { go func() { - now, _ := time.Now().MarshalText() + now, _ := time.Now().UTC().MarshalText() s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) // send the ping flag and get the latency first ticker := time.NewTicker(time.Second * 15) @@ -166,7 +166,7 @@ func (s *Mux) ping() { if (math.MaxInt32 - s.id) < 10000 { s.id = 0 } - now, _ := time.Now().MarshalText() + now, _ := time.Now().UTC().MarshalText() s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) if s.pingOk > 10 && s.connType == "kcp" { s.Close() @@ -188,8 +188,11 @@ func (s *Mux) pingReturn() { break } _ = now.UnmarshalText(data) - s.latency = time.Since(now).Seconds() - s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, nil) + s.latency = time.Now().UTC().Sub(now).Seconds() / 2 + //logs.Warn("latency", s.latency) + if s.latency <= 0 { + logs.Warn("latency err", s.latency) + } } }() } @@ -214,9 +217,10 @@ func (s *Mux) readSession() { s.sendInfo(common.MUX_NEW_CONN_OK, connection.connId, nil) continue case common.MUX_PING_FLAG: //ping - s.pingCh <- pack.Content + s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, pack.Content) continue case common.MUX_PING_RETURN: + s.pingCh <- pack.Content continue } if connection, ok := s.connMap.Get(pack.Id); ok && !connection.isClose { From 4c8d7b0738f492db4904c5f9c976359178dc93e0 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Tue, 8 Oct 2019 21:41:25 +0800 Subject: [PATCH 13/34] reduce memory allocate --- lib/common/const.go | 2 +- lib/common/pool.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/common/const.go b/lib/common/const.go index 95364a2..b4eac1a 100644 --- a/lib/common/const.go +++ b/lib/common/const.go @@ -48,5 +48,5 @@ const ( MUX_CONN_CLOSE MUX_PING_RETURN MUX_PING int32 = -1 - MAXIMUM_SEGMENT_SIZE = 4096 - 16 - 32 - 32 - 8 + MAXIMUM_SEGMENT_SIZE = PoolSizeWindow ) diff --git a/lib/common/pool.go b/lib/common/pool.go index 98da5d3..24efc60 100644 --- a/lib/common/pool.go +++ b/lib/common/pool.go @@ -9,7 +9,8 @@ const PoolSize = 64 * 1024 const PoolSizeSmall = 100 const PoolSizeUdp = 1472 const PoolSizeCopy = 32 << 10 -const PoolSizeWindow = 1<<16 - 1 +const PoolSizeBuffer = 4096 +const PoolSizeWindow = PoolSizeBuffer - 16 - 32 - 32 - 8 var BufPool = sync.Pool{ New: func() interface{} { @@ -92,18 +93,18 @@ type windowBufferPool struct { func (Self *windowBufferPool) New() { Self.pool = sync.Pool{ New: func() interface{} { - return make([]byte, 0, PoolSizeWindow) + return make([]byte, PoolSizeWindow, PoolSizeWindow) }, } } func (Self *windowBufferPool) Get() (buf []byte) { buf = Self.pool.Get().([]byte) - return buf[:0] + return buf[:PoolSizeWindow] } func (Self *windowBufferPool) Put(x []byte) { - if cap(x) == PoolSizeWindow { + if len(x) == PoolSizeWindow { Self.pool.Put(x[:PoolSizeWindow]) // make buf to full } else { x = nil @@ -117,7 +118,7 @@ type bufferPool struct { func (Self *bufferPool) New() { Self.pool = sync.Pool{ New: func() interface{} { - return new(bytes.Buffer) + return bytes.NewBuffer(make([]byte, 0, PoolSizeBuffer)) }, } } @@ -146,13 +147,12 @@ func (Self *muxPackagerPool) New() { func (Self *muxPackagerPool) Get() *MuxPackager { pack := Self.pool.Get().(*MuxPackager) - buf := CopyBuff.Get() - pack.Content = buf + pack.Content = WindowBuff.Get() return pack } func (Self *muxPackagerPool) Put(pack *MuxPackager) { - CopyBuff.Put(pack.Content) + WindowBuff.Put(pack.Content) Self.pool.Put(pack) } From 1f8e4410906a8e5a557535078818e22dff7bf35f Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sat, 12 Oct 2019 22:56:37 +0800 Subject: [PATCH 14/34] multiple changes --- README.md | 11 ++++++++- client/client.go | 5 ++++ client/control.go | 9 +++++-- cmd/nps/nps.go | 2 +- lib/common/const.go | 1 + lib/common/netpackager.go | 9 +++++-- lib/common/pool.go | 11 ++------- lib/common/util.go | 3 +++ lib/install/install.go | 38 +++++++++++++++++++++++++--- lib/mux/conn.go | 8 ++++++ lib/mux/mux.go | 17 +++++++++++-- lib/mux/mux_test.go | 52 +++++++++++++++++++++++++++++++++++++++ lib/mux/queue.go | 39 ++++++++++++++++++++--------- lib/version/version.go | 4 +-- 14 files changed, 176 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 1dd7f77..f56c32a 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,9 @@ nps是一款轻量级、高性能、功能强大的**内网穿透**代理服务 - 在刚才创建的客户端隧道管理中添加一条socks5代理，填写监听的端口（8003），保存。 - 在外网环境的本机配置socks5代理(例如使用proxifier进行全局代理)，ip为公网服务器ip（1.1.1.1），端口为填写的监听端口(8003)，即可畅享内网了 +**注意** +经过socks5代理，当收到socks5数据包时socket已经是accept状态。表现是扫描端口全open，建立连接后短时间关闭。若想同内网表现一致，建议远程连接一台设备。 + ### http正向代理 **适用范围：** 在外网环境下使用http正向代理访问内网站点 @@ -375,7 +378,13 @@ server { ``` (./nps|nps.exe) install ``` -安装成功后，对于linux，darwin，将会把配置文件和静态文件放置于/etc/nps/，并将可执行文件nps复制到/usr/bin/nps或者/usr/local/bin/nps，安装成功后可在任何位置执行 +安装成功后，对于linux，darwin，将会把配置文件和静态文件放置于/etc/nps/，并将可执行文件nps复制到/usr/bin/nps或者/usr/local/bin/nps，安装成功后可在任何位置执行，同时也会添加systemd配置。 + +``` +sudo systemctl enable|disable|start|stop|restart|status nps +``` +systemd，带有开机自启，自动重启配置，当进程结束后15秒会启动，日志输出至/var/log/nps/nps.log。 +建议采用此方式启动，能够捕获panic信息，便于排查问题。 ``` nps test|start|stop|restart|status diff --git a/client/client.go b/client/client.go index 52da907..5bd0b01 100755 --- a/client/client.go +++ b/client/client.go @@ -49,6 +49,11 @@ retry: time.Sleep(time.Second * 5) goto retry } + if c == nil { + logs.Error("Error data from server, and will be reconnected in five seconds") + time.Sleep(time.Second * 5) + goto retry + } logs.Info("Successful connection with server %s", s.svrAddr) //monitor the connection go s.ping() diff --git a/client/control.go b/client/control.go index 5673f14..3260113 100644 --- a/client/control.go +++ b/client/control.go @@ -223,8 +223,13 @@ func NewConn(tp string, vkey string, server string, connType string, proxyUrl st if _, err := c.Write([]byte(crypt.Md5(version.GetVersion()))); err != nil { return nil, err } - if b, err := c.GetShortContent(32); err != nil || crypt.Md5(version.GetVersion()) != string(b) { - logs.Error("The client does not match the server version. The current version of the client is", version.GetVersion()) + b, err := c.GetShortContent(32) + if err != nil { + logs.Error(err) + return nil, err + } + if crypt.Md5(version.GetVersion()) != string(b) { + logs.Error("The client does not match the server version. The current core version of the client is", version.GetVersion()) return nil, err } if _, err := c.Write([]byte(common.Getverifyval(vkey))); err != nil { diff --git a/cmd/nps/nps.go b/cmd/nps/nps.go index f66fe66..22835a2 100644 --- a/cmd/nps/nps.go +++ b/cmd/nps/nps.go @@ -61,7 +61,7 @@ func main() { logs.Error("Getting bridge_port error", err) os.Exit(0) } - logs.Info("the version of server is %s ,allow client version to be %s", version.VERSION, version.GetVersion()) + logs.Info("the version of server is %s ,allow client core version to be %s", version.VERSION, version.GetVersion()) connection.InitConnectionService() crypt.InitTls(filepath.Join(common.GetRunPath(), "conf", "server.pem"), filepath.Join(common.GetRunPath(), "conf", "server.key")) tool.InitAllowPort() diff --git a/lib/common/const.go b/lib/common/const.go index b4eac1a..f57ce4f 100644 --- a/lib/common/const.go +++ b/lib/common/const.go @@ -49,4 +49,5 @@ const ( MUX_PING_RETURN MUX_PING int32 = -1 MAXIMUM_SEGMENT_SIZE = PoolSizeWindow + MAXIMUM_WINDOW_SIZE = 1<<31 - 1 ) diff --git a/lib/common/netpackager.go b/lib/common/netpackager.go index 69ce96a..567a48f 100644 --- a/lib/common/netpackager.go +++ b/lib/common/netpackager.go @@ -158,8 +158,10 @@ func (Self *MuxPackager) NewPac(flag uint8, id int32, content ...interface{}) (e Self.Flag = flag Self.Id = id switch flag { - case MUX_NEW_MSG, MUX_NEW_MSG_PART, MUX_PING_FLAG, MUX_PING_RETURN: + case MUX_PING_FLAG, MUX_PING_RETURN, MUX_NEW_MSG, MUX_NEW_MSG_PART: + Self.Content = WindowBuff.Get() err = Self.BasePackager.NewPac(content...) + //logs.Warn(Self.Length, string(Self.Content)) case MUX_MSG_SEND_OK: // MUX_MSG_SEND_OK contains two data switch content[0].(type) { @@ -190,6 +192,7 @@ func (Self *MuxPackager) Pack(writer io.Writer) (err error) { switch Self.Flag { case MUX_NEW_MSG, MUX_NEW_MSG_PART, MUX_PING_FLAG, MUX_PING_RETURN: err = Self.BasePackager.Pack(writer) + WindowBuff.Put(Self.Content) case MUX_MSG_SEND_OK: err = binary.Write(writer, binary.LittleEndian, Self.Window) if err != nil { @@ -201,7 +204,6 @@ func (Self *MuxPackager) Pack(writer io.Writer) (err error) { } func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { - Self.BasePackager.clean() // also clean the content err = binary.Read(reader, binary.LittleEndian, &Self.Flag) if err != nil { return @@ -212,7 +214,10 @@ func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { } switch Self.Flag { case MUX_NEW_MSG, MUX_NEW_MSG_PART, MUX_PING_FLAG, MUX_PING_RETURN: + Self.Content = WindowBuff.Get() // need get a window buf from pool + Self.BasePackager.clean() // also clean the content err = Self.BasePackager.UnPack(reader) + //logs.Warn("unpack", Self.Length, string(Self.Content)) case MUX_MSG_SEND_OK: err = binary.Read(reader, binary.LittleEndian, &Self.Window) if err != nil { diff --git a/lib/common/pool.go b/lib/common/pool.go index 24efc60..240f7f9 100644 --- a/lib/common/pool.go +++ b/lib/common/pool.go @@ -104,11 +104,7 @@ func (Self *windowBufferPool) Get() (buf []byte) { } func (Self *windowBufferPool) Put(x []byte) { - if len(x) == PoolSizeWindow { - Self.pool.Put(x[:PoolSizeWindow]) // make buf to full - } else { - x = nil - } + Self.pool.Put(x[:PoolSizeWindow]) // make buf to full } type bufferPool struct { @@ -146,13 +142,10 @@ func (Self *muxPackagerPool) New() { } func (Self *muxPackagerPool) Get() *MuxPackager { - pack := Self.pool.Get().(*MuxPackager) - pack.Content = WindowBuff.Get() - return pack + return Self.pool.Get().(*MuxPackager) } func (Self *muxPackagerPool) Put(pack *MuxPackager) { - WindowBuff.Put(pack.Content) Self.pool.Put(pack) } diff --git a/lib/common/util.go b/lib/common/util.go index dc9afbe..e3dfb4f 100755 --- a/lib/common/util.go +++ b/lib/common/util.go @@ -268,6 +268,9 @@ func CopyBuffer(dst io.Writer, src io.Reader) (written int64, err error) { defer CopyBuff.Put(buf) for { nr, er := src.Read(buf) + //if len(pr)>0 && pr[0] && nr > 50 { + // logs.Warn(string(buf[:50])) + //} if nr > 0 { nw, ew := dst.Write(buf[0:nr]) if nw > 0 { diff --git a/lib/install/install.go b/lib/install/install.go index 56f3cc5..24af9b9 100644 --- a/lib/install/install.go +++ b/lib/install/install.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "io/ioutil" "log" "os" "path/filepath" @@ -13,6 +14,23 @@ import ( ) func InstallNps() { + unit := `[Unit] +Description=nps - convenient proxy server +Documentation=https://github.com/cnlh/nps/ +After=network-online.target remote-fs.target nss-lookup.target +Wants=network-online.target` + service := `[Service] +Type=simple +KillMode=process +Restart=always +RestartSec=15s +StandardOutput=append:/var/log/nps/nps.log +ExecStartPre=/bin/echo 'Starting nps' +ExecStopPost=/bin/echo 'Stopping nps' +ExecStart=` + install := `[Install] +WantedBy=multi-user.target` + path := common.GetInstallPath() if common.FileExists(path) { log.Fatalf("the path %s has exist, does not support install", path) @@ -35,21 +53,35 @@ func InstallNps() { log.Fatalln(err) } else { os.Chmod("/usr/local/bin/nps", 0755) + service += "/usr/local/bin/nps" log.Println("Executable files have been copied to", "/usr/local/bin/nps") } } else { os.Chmod("/usr/bin/nps", 0755) + service += "/usr/bin/nps" log.Println("Executable files have been copied to", "/usr/bin/nps") } - + systemd := unit + "\n\n" + service + "\n\n" + install + _ = os.Remove("/usr/lib/systemd/system/nps.service") + err := ioutil.WriteFile("/usr/lib/systemd/system/nps.service", []byte(systemd), 0644) + if err != nil { + log.Println("Write systemd service err ", err) + } + _ = os.Mkdir("/var/log/nps", 644) } log.Println("install ok!") log.Println("Static files and configuration files in the current directory will be useless") log.Println("The new configuration file is located in", path, "you can edit them") if !common.IsWindows() { - log.Println("You can start with nps test|start|stop|restart|status anywhere") + log.Println(`You can start with: +sudo systemctl enable|disable|start|stop|restart|status nps +or: +nps test|start|stop|restart|status +anywhere!`) } else { - log.Println("You can copy executable files to any directory and start working with nps.exe test|start|stop|restart|status") + log.Println(`You can copy executable files to any directory and start working with: +nps.exe test|start|stop|restart|status +now!`) } } func MkidrDirAll(path string, v ...string) { diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 5dd69ea..c4b47f3 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -183,6 +183,10 @@ func (Self *ReceiveWindow) CalcSize() { n = Self.bufQueue.Len() } // set the minimal size + if n > common.MAXIMUM_WINDOW_SIZE { + n = common.MAXIMUM_WINDOW_SIZE + } + // set the maximum size //logs.Warn("n", n) Self.maxSize = n Self.count = -5 @@ -248,6 +252,10 @@ copyData: l = 0 Self.bw.EndRead() Self.sendStatus(id) + if Self.off == uint32(Self.element.l) { + //logs.Warn("put the element end ", string(Self.element.buf[:15])) + common.WindowBuff.Put(Self.element.buf) + } if pOff < len(p) && Self.element.part { // element is a part of the segments, trying to fill up buf p goto copyData diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 9023b82..529b7dc 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -25,6 +25,7 @@ type Mux struct { pingOk int latency float64 pingCh chan []byte + pingTimer *time.Timer connType string writeQueue PriorityQueue bufCh chan *bytes.Buffer @@ -42,6 +43,7 @@ func NewMux(c net.Conn, connType string) *Mux { connType: connType, bufCh: make(chan *bytes.Buffer), pingCh: make(chan []byte), + pingTimer: time.NewTimer(15 * time.Second), } m.writeQueue.New() //read session by flag @@ -119,6 +121,7 @@ func (s *Mux) packBuf() { common.BuffPool.Put(buffer) break } + //logs.Warn(buffer.String()) select { case s.bufCh <- buffer: case <-s.closeChan: @@ -153,7 +156,7 @@ func (s *Mux) ping() { now, _ := time.Now().UTC().MarshalText() s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) // send the ping flag and get the latency first - ticker := time.NewTicker(time.Second * 15) + ticker := time.NewTicker(time.Second * 5) for { if s.IsClose { ticker.Stop() @@ -168,6 +171,10 @@ func (s *Mux) ping() { } now, _ := time.Now().UTC().MarshalText() s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) + if !s.pingTimer.Stop() { + <-s.pingTimer.C + } + s.pingTimer.Reset(15 * time.Second) if s.pingOk > 10 && s.connType == "kcp" { s.Close() break @@ -186,10 +193,15 @@ func (s *Mux) pingReturn() { case data = <-s.pingCh: case <-s.closeChan: break + case <-s.pingTimer.C: + logs.Error("mux: ping time out") + s.Close() + break } _ = now.UnmarshalText(data) s.latency = time.Now().UTC().Sub(now).Seconds() / 2 - //logs.Warn("latency", s.latency) + logs.Warn("latency", s.latency) + common.WindowBuff.Put(data) if s.latency <= 0 { logs.Warn("latency err", s.latency) } @@ -218,6 +230,7 @@ func (s *Mux) readSession() { continue case common.MUX_PING_FLAG: //ping s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, pack.Content) + common.WindowBuff.Put(pack.Content) continue case common.MUX_PING_RETURN: s.pingCh <- pack.Content diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index c7b10e0..abc4eb4 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -1,8 +1,11 @@ package mux import ( + "bufio" + "fmt" "net" "net/http" + "net/http/httputil" _ "net/http/pprof" "sync" "testing" @@ -37,6 +40,7 @@ func TestNewMux(t *testing.T) { c2, err := net.Dial("tcp", "127.0.0.1:80") if err != nil { logs.Warn(err) + c.Close() continue } go func(c2 net.Conn, c net.Conn) { @@ -107,6 +111,9 @@ func TestNewMux(t *testing.T) { } }() + time.Sleep(time.Second * 5) + //go test_request() + for { time.Sleep(time.Second * 5) } @@ -135,6 +142,51 @@ func client() { } } +func test_request() { + conn, _ := net.Dial("tcp", "127.0.0.1:7777") + for { + conn.Write([]byte(`GET /videojs5/video.js HTTP/1.1 +Host: 127.0.0.1:7777 +Connection: keep-alive + + +`)) + r, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil { + logs.Warn("close by read response err", err) + break + } + logs.Warn("read response success", r) + b, err := httputil.DumpResponse(r, true) + if err != nil { + logs.Warn("close by dump response err", err) + break + } + fmt.Println(string(b[:20]), err) + time.Sleep(time.Second) + } +} + +func test_raw() { + conn, _ := net.Dial("tcp", "127.0.0.1:7777") + for { + conn.Write([]byte(`GET /videojs5/test HTTP/1.1 +Host: 127.0.0.1:7777 +Connection: keep-alive + + +`)) + buf := make([]byte, 1000000) + n, err := conn.Read(buf) + if err != nil { + logs.Warn("close by read response err", err) + break + } + logs.Warn(n, string(buf[:50]), "\n--------------\n", string(buf[n-50:n])) + time.Sleep(time.Second) + } +} + func TestNewConn(t *testing.T) { buf := common.GetBufPoolCopy() logs.Warn(len(buf), cap(buf)) diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 5a57151..a835e2a 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -51,15 +51,23 @@ func (Self *PriorityQueue) New() { func (Self *PriorityQueue) Push(packager *common.MuxPackager) { Self.mutex.Lock() - if Self.popWait { - defer Self.allowPop() - } - if packager.Flag == common.MUX_CONN_CLOSE { - Self.insert(packager) // the close package may need priority, - // prevent wait too long to close - } else { + switch packager.Flag { + case common.MUX_PING_FLAG, common.MUX_PING_RETURN: + Self.list.PushFront(packager) + // the ping package need highest priority + // prevent ping calculation error + case common.MUX_CONN_CLOSE: + Self.insert(packager) + // the close package may need priority too, set second + // prevent wait too long to close conn + default: Self.list.PushBack(packager) } + if Self.popWait { + Self.mutex.Unlock() + Self.allowPop() + return + } Self.mutex.Unlock() return } @@ -68,7 +76,14 @@ func (Self *PriorityQueue) insert(packager *common.MuxPackager) { element := Self.list.Back() for { if element == nil { // PriorityQueue dose not have any of msg package with this close package id - Self.list.PushFront(packager) // insert close package to first + element = Self.list.Front() + if element != nil { + Self.list.InsertAfter(packager, element) + // insert close package to second + } else { + Self.list.PushFront(packager) + // list is empty, push to front + } break } if element.Value.(*common.MuxPackager).Flag == common.MUX_NEW_MSG && @@ -136,11 +151,13 @@ func (Self *FIFOQueue) New() { func (Self *FIFOQueue) Push(element *ListElement) { Self.mutex.Lock() - if Self.popWait { - defer Self.allowPop() - } Self.list = append(Self.list, element) Self.length += uint32(element.l) + if Self.popWait { + Self.mutex.Unlock() + Self.allowPop() + return + } Self.mutex.Unlock() return } diff --git a/lib/version/version.go b/lib/version/version.go index 902b30b..4cc0532 100644 --- a/lib/version/version.go +++ b/lib/version/version.go @@ -1,8 +1,8 @@ package version -const VERSION = "0.23.2" +const VERSION = "0.23.3" // Compulsory minimum version, Minimum downward compatibility to this version func GetVersion() string { - return "0.21.0" + return "0.22.0" } From f5d5f633660d7ffb1325dbcb3a18bd2797431ea2 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sun, 13 Oct 2019 22:45:40 +0800 Subject: [PATCH 15/34] change slide window bandwidth calculation --- lib/common/netpackager.go | 16 ++-- lib/mux/conn.go | 160 +++++++++++++++++++++----------------- lib/mux/map.go | 4 + lib/mux/mux.go | 67 ++++++++++++++-- 4 files changed, 163 insertions(+), 84 deletions(-) diff --git a/lib/common/netpackager.go b/lib/common/netpackager.go index 567a48f..91eeb98 100644 --- a/lib/common/netpackager.go +++ b/lib/common/netpackager.go @@ -65,8 +65,9 @@ func (Self *BasePackager) Pack(writer io.Writer) (err error) { //Unpack 会导致传入的数字类型转化成float64！！ //主要原因是json unmarshal并未传入正确的数据类型 -func (Self *BasePackager) UnPack(reader io.Reader) (err error) { +func (Self *BasePackager) UnPack(reader io.Reader) (n uint16, err error) { Self.clean() + n += 2 // uint16 err = binary.Read(reader, binary.LittleEndian, &Self.Length) if err != nil { return @@ -80,6 +81,7 @@ func (Self *BasePackager) UnPack(reader io.Reader) (err error) { // err = io.ErrUnexpectedEOF //} err = binary.Read(reader, binary.LittleEndian, Self.Content) + n += Self.Length return } @@ -137,12 +139,13 @@ func (Self *ConnPackager) Pack(writer io.Writer) (err error) { return } -func (Self *ConnPackager) UnPack(reader io.Reader) (err error) { +func (Self *ConnPackager) UnPack(reader io.Reader) (n uint16, err error) { err = binary.Read(reader, binary.LittleEndian, &Self.ConnType) if err != nil && err != io.EOF { return } - err = Self.BasePackager.UnPack(reader) + n, err = Self.BasePackager.UnPack(reader) + n += 2 return } @@ -203,7 +206,7 @@ func (Self *MuxPackager) Pack(writer io.Writer) (err error) { return } -func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { +func (Self *MuxPackager) UnPack(reader io.Reader) (n uint16, err error) { err = binary.Read(reader, binary.LittleEndian, &Self.Flag) if err != nil { return @@ -216,14 +219,17 @@ func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { case MUX_NEW_MSG, MUX_NEW_MSG_PART, MUX_PING_FLAG, MUX_PING_RETURN: Self.Content = WindowBuff.Get() // need get a window buf from pool Self.BasePackager.clean() // also clean the content - err = Self.BasePackager.UnPack(reader) + n, err = Self.BasePackager.UnPack(reader) //logs.Warn("unpack", Self.Length, string(Self.Content)) case MUX_MSG_SEND_OK: err = binary.Read(reader, binary.LittleEndian, &Self.Window) if err != nil { return } + n += 4 // uint32 err = binary.Read(reader, binary.LittleEndian, &Self.ReadLength) + n += 4 // uint32 } + n += 5 //uint8 int32 return } diff --git a/lib/mux/conn.go b/lib/mux/conn.go index c4b47f3..2d519cb 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -3,7 +3,6 @@ package mux import ( "errors" "io" - "math" "net" "sync" "time" @@ -137,8 +136,8 @@ type ReceiveWindow struct { readWait bool windowFull bool count int8 - bw *bandwidth - once sync.Once + //bw *bandwidth + once sync.Once window } @@ -146,7 +145,7 @@ func (Self *ReceiveWindow) New(mux *Mux) { // initial a window for receive Self.readOp = make(chan struct{}) Self.bufQueue.New() - Self.bw = new(bandwidth) + //Self.bw = new(bandwidth) Self.element = new(ListElement) Self.maxSize = 8192 Self.mux = mux @@ -175,7 +174,7 @@ func (Self *ReceiveWindow) CalcSize() { // calculating maximum receive window size if Self.count == 0 { //logs.Warn("ping, bw", Self.mux.latency, Self.bw.Get()) - n := uint32(2 * Self.mux.latency * Self.bw.Get()) + n := uint32(2 * Self.mux.latency * Self.mux.bw.Get() * 1.5 / float64(Self.mux.connMap.Size())) if n < 8192 { n = 8192 } @@ -183,13 +182,16 @@ func (Self *ReceiveWindow) CalcSize() { n = Self.bufQueue.Len() } // set the minimal size + if n > 2*Self.maxSize { + n = 2 * Self.maxSize + } if n > common.MAXIMUM_WINDOW_SIZE { n = common.MAXIMUM_WINDOW_SIZE } // set the maximum size //logs.Warn("n", n) Self.maxSize = n - Self.count = -5 + Self.count = -10 } Self.count += 1 } @@ -225,7 +227,7 @@ func (Self *ReceiveWindow) Read(p []byte, id int32) (n int, err error) { l := 0 //logs.Warn("receive window read off, element.l", Self.off, Self.element.l) copyData: - Self.bw.StartRead() + //Self.bw.StartRead() if Self.off == uint32(Self.element.l) { // on the first Read method invoked, Self.off and Self.element.l // both zero value @@ -241,7 +243,7 @@ copyData: //logs.Warn("pop element", Self.element.l, Self.element.part) } l = copy(p[pOff:], Self.element.buf[Self.off:]) - Self.bw.SetCopySize(l) + //Self.bw.SetCopySize(l) pOff += l Self.off += uint32(l) Self.bufQueue.mutex.Lock() @@ -250,7 +252,7 @@ copyData: Self.bufQueue.mutex.Unlock() n += l l = 0 - Self.bw.EndRead() + //Self.bw.EndRead() Self.sendStatus(id) if Self.off == uint32(Self.element.l) { //logs.Warn("put the element end ", string(Self.element.buf[:15])) @@ -469,65 +471,81 @@ func (Self *SendWindow) SetTimeOut(t time.Time) { Self.timeout = t } -type bandwidth struct { - lastReadStart time.Time - readStart time.Time - readEnd time.Time - bufLength int - lastBufLength int - count int8 - readBW float64 - writeBW float64 -} - -func (Self *bandwidth) StartRead() { - Self.lastReadStart, Self.readStart = Self.readStart, time.Now() -} - -func (Self *bandwidth) EndRead() { - if !Self.lastReadStart.IsZero() { - if Self.count == 0 { - Self.calcWriteBandwidth() - } - } - Self.readEnd = time.Now() - if Self.count == 0 { - Self.calcReadBandwidth() - Self.count = -3 - } - Self.count += 1 -} - -func (Self *bandwidth) SetCopySize(n int) { - // must be invoke between StartRead and EndRead - Self.lastBufLength, Self.bufLength = Self.bufLength, n -} - -func (Self *bandwidth) calcReadBandwidth() { - // Bandwidth between nps and npc - readTime := Self.readEnd.Sub(Self.readStart) - Self.readBW = float64(Self.bufLength) / readTime.Seconds() - //logs.Warn("calc read bw", Self.bufLength, readTime.Seconds()) -} - -func (Self *bandwidth) calcWriteBandwidth() { - // Bandwidth between nps and user, npc and application - //logs.Warn("calc write bw") - writeTime := Self.readEnd.Sub(Self.lastReadStart) - Self.writeBW = float64(Self.lastBufLength) / writeTime.Seconds() -} - -func (Self *bandwidth) Get() (bw float64) { - // The zero value, 0 for numeric types - if Self.writeBW == 0 && Self.readBW == 0 { - //logs.Warn("bw both 0") - return 100 - } - if Self.writeBW == 0 && Self.readBW != 0 { - return Self.readBW - } - if Self.readBW == 0 && Self.writeBW != 0 { - return Self.writeBW - } - return math.Min(Self.readBW, Self.writeBW) -} +//type bandwidth struct { +// readStart time.Time +// lastReadStart time.Time +// readEnd time.Time +// lastReadEnd time.Time +// bufLength int +// lastBufLength int +// count int8 +// readBW float64 +// writeBW float64 +// readBandwidth float64 +//} +// +//func (Self *bandwidth) StartRead() { +// Self.lastReadStart, Self.readStart = Self.readStart, time.Now() +// if !Self.lastReadStart.IsZero() { +// if Self.count == -5 { +// Self.calcBandWidth() +// } +// } +//} +// +//func (Self *bandwidth) EndRead() { +// Self.lastReadEnd, Self.readEnd = Self.readEnd, time.Now() +// if Self.count == -5 { +// Self.calcWriteBandwidth() +// } +// if Self.count == 0 { +// Self.calcReadBandwidth() +// Self.count = -6 +// } +// Self.count += 1 +//} +// +//func (Self *bandwidth) SetCopySize(n int) { +// // must be invoke between StartRead and EndRead +// Self.lastBufLength, Self.bufLength = Self.bufLength, n +//} +//// calculating +//// start end start end +//// read read +//// write +// +//func (Self *bandwidth) calcBandWidth() { +// t := Self.readStart.Sub(Self.lastReadStart) +// if Self.lastBufLength >= 32768 { +// Self.readBandwidth = float64(Self.lastBufLength) / t.Seconds() +// } +//} +// +//func (Self *bandwidth) calcReadBandwidth() { +// // Bandwidth between nps and npc +// readTime := Self.readEnd.Sub(Self.readStart) +// Self.readBW = float64(Self.bufLength) / readTime.Seconds() +// //logs.Warn("calc read bw", Self.readBW, Self.bufLength, readTime.Seconds()) +//} +// +//func (Self *bandwidth) calcWriteBandwidth() { +// // Bandwidth between nps and user, npc and application +// writeTime := Self.readStart.Sub(Self.lastReadEnd) +// Self.writeBW = float64(Self.lastBufLength) / writeTime.Seconds() +// //logs.Warn("calc write bw", Self.writeBW, Self.bufLength, writeTime.Seconds()) +//} +// +//func (Self *bandwidth) Get() (bw float64) { +// // The zero value, 0 for numeric types +// if Self.writeBW == 0 && Self.readBW == 0 { +// //logs.Warn("bw both 0") +// return 100 +// } +// if Self.writeBW == 0 && Self.readBW != 0 { +// return Self.readBW +// } +// if Self.readBW == 0 && Self.writeBW != 0 { +// return Self.writeBW +// } +// return Self.readBandwidth +//} diff --git a/lib/mux/map.go b/lib/mux/map.go index 0801201..8f07dee 100644 --- a/lib/mux/map.go +++ b/lib/mux/map.go @@ -20,6 +20,10 @@ func NewConnMap() *connMap { return connMap } +func (s *connMap) Size() (n int) { + return len(s.connMap) +} + func (s *connMap) Get(id int32) (*conn, bool) { s.Lock() defer s.Unlock() diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 529b7dc..42aa8c8 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -24,6 +24,7 @@ type Mux struct { IsClose bool pingOk int latency float64 + bw *bandwidth pingCh chan []byte pingTimer *time.Timer connType string @@ -37,8 +38,9 @@ func NewMux(c net.Conn, connType string) *Mux { conn: c, connMap: NewConnMap(), id: 0, - closeChan: make(chan struct{}), + closeChan: make(chan struct{}, 3), newConnCh: make(chan *conn), + bw: new(bandwidth), IsClose: false, connType: connType, bufCh: make(chan *bytes.Buffer), @@ -91,6 +93,9 @@ func (s *Mux) Addr() net.Addr { } func (s *Mux) sendInfo(flag uint8, id int32, data ...interface{}) { + if s.IsClose { + return + } var err error pack := common.MuxPack.Get() err = pack.NewPac(flag, id, data...) @@ -160,6 +165,9 @@ func (s *Mux) ping() { for { if s.IsClose { ticker.Stop() + if !s.pingTimer.Stop() { + <-s.pingTimer.C + } break } select { @@ -189,6 +197,9 @@ func (s *Mux) pingReturn() { var now time.Time var data []byte for { + if s.IsClose { + break + } select { case data = <-s.pingCh: case <-s.closeChan: @@ -199,12 +210,12 @@ func (s *Mux) pingReturn() { break } _ = now.UnmarshalText(data) - s.latency = time.Now().UTC().Sub(now).Seconds() / 2 - logs.Warn("latency", s.latency) - common.WindowBuff.Put(data) - if s.latency <= 0 { - logs.Warn("latency err", s.latency) + latency := time.Now().UTC().Sub(now).Seconds() / 2 + if latency < 0.5 && latency > 0 { + s.latency = latency } + //logs.Warn("latency", s.latency) + common.WindowBuff.Put(data) } }() } @@ -212,14 +223,18 @@ func (s *Mux) pingReturn() { func (s *Mux) readSession() { go func() { pack := common.MuxPack.Get() + var l uint16 + var err error for { if s.IsClose { break } pack = common.MuxPack.Get() - if pack.UnPack(s.conn) != nil { + s.bw.StartRead() + if l, err = pack.UnPack(s.conn); err != nil { break } + s.bw.SetCopySize(l) s.pingOk = 0 switch pack.Flag { case common.MUX_NEW_CONN: //new connection @@ -239,7 +254,7 @@ func (s *Mux) readSession() { if connection, ok := s.connMap.Get(pack.Id); ok && !connection.isClose { switch pack.Flag { case common.MUX_NEW_MSG, common.MUX_NEW_MSG_PART: //new msg from remote connection - err := s.newMsg(connection, pack) + err = s.newMsg(connection, pack) if err != nil { connection.Close() } @@ -299,6 +314,7 @@ func (s *Mux) Close() error { s.connMap.Close() s.closeChan <- struct{}{} s.closeChan <- struct{}{} + s.closeChan <- struct{}{} close(s.newConnCh) return s.conn.Close() } @@ -311,3 +327,38 @@ func (s *Mux) getId() (id int32) { } return } + +type bandwidth struct { + readStart time.Time + lastReadStart time.Time + bufLength uint16 + readBandwidth float64 +} + +func (Self *bandwidth) StartRead() { + if Self.readStart.IsZero() { + Self.readStart = time.Now() + } + if Self.bufLength >= 16384 { + Self.lastReadStart, Self.readStart = Self.readStart, time.Now() + Self.calcBandWidth() + } +} + +func (Self *bandwidth) SetCopySize(n uint16) { + Self.bufLength += n +} + +func (Self *bandwidth) calcBandWidth() { + t := Self.readStart.Sub(Self.lastReadStart) + Self.readBandwidth = float64(Self.bufLength) / t.Seconds() + Self.bufLength = 0 +} + +func (Self *bandwidth) Get() (bw float64) { + // The zero value, 0 for numeric types + if Self.readBandwidth <= 0 { + Self.readBandwidth = 100 + } + return Self.readBandwidth +} From 18ca5d04ccf2733c818160addb2f6526d483db5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=B2=B3?= Date: Mon, 14 Oct 2019 23:44:07 +0800 Subject: [PATCH 16/34] connection trace web server --- lib/mux/web.go | 97 +++++++++++++++++++++++++++++++++++++++++++++ lib/mux/web_test.go | 7 ++++ 2 files changed, 104 insertions(+) create mode 100644 lib/mux/web.go create mode 100644 lib/mux/web_test.go diff --git a/lib/mux/web.go b/lib/mux/web.go new file mode 100644 index 0000000..0ebe8e7 --- /dev/null +++ b/lib/mux/web.go @@ -0,0 +1,97 @@ +package mux + +import ( + "fmt" + "net/http" + "sort" + "strconv" + "time" +) + +type connLog struct { + startTime time.Time + isClose bool + logs []string +} + +var m map[int]*connLog + +var copyMap map[int]*connLog + +func deepCopyMap() { + copyMap = make(map[int]*connLog) + for k, v := range m { + copyMap[k] = &connLog{ + startTime: v.startTime, + isClose: v.isClose, + logs: v.logs, + } + } +} + +//func init() { +// m = make(map[int]*connLog) +// m[0] = &connLog{ +// startTime: time.Now(), +// isClose: false, +// logs: []string{"111", "222", "333"}, +// } +// m[1] = &connLog{ +// startTime: time.Now(), +// isClose: false, +// logs: []string{"111", "222", "333", "444"}, +// } +// m[2] = &connLog{ +// startTime: time.Now(), +// isClose: true, +// logs: []string{"111", "222", "333", "555"}, +// } +//} + +type IntSlice []int + +func (s IntSlice) Len() int { return len(s) } + +func (s IntSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func (s IntSlice) Less(i, j int) bool { return s[i] < s[j] } + +func NewLogServer() { + http.HandleFunc("/", index) + http.HandleFunc("/detail", detail) + http.HandleFunc("/stash", stash) + fmt.Println(http.ListenAndServe(":8899", nil)) +} + +func stash(w http.ResponseWriter, r *http.Request) { + deepCopyMap() + w.Write([]byte("ok")) +} + +func index(w http.ResponseWriter, r *http.Request) { + var keys []int + for k := range copyMap { + keys = append(keys, k) + } + sort.Sort(IntSlice(keys)) + var s string + for v := range keys { + connL := copyMap[v] + s += "" + strconv.Itoa(v) + "----------" + s += strconv.Itoa(int(time.Now().Unix()-connL.startTime.Unix())) + "s----------" + s += strconv.FormatBool(connL.isClose) + s += "
" + } + w.Write([]byte(s)) +} + +func detail(w http.ResponseWriter, r *http.Request) { + id := r.FormValue("id") + i, _ := strconv.Atoi(id) + v, _ := copyMap[i] + var s string + for _, vv := range v.logs { + s += "

" + vv + "

" + } + w.Write([]byte(s)) +} diff --git a/lib/mux/web_test.go b/lib/mux/web_test.go new file mode 100644 index 0000000..91a0430 --- /dev/null +++ b/lib/mux/web_test.go @@ -0,0 +1,7 @@ +package mux + +import "testing" + +func TestWeb(t *testing.T) { + NewLogServer() +} From d23ed2126dc97eef6648a7839771953bdd615372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=B2=B3?= Date: Mon, 14 Oct 2019 23:46:00 +0800 Subject: [PATCH 17/34] change now time --- lib/mux/web.go | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/lib/mux/web.go b/lib/mux/web.go index 0ebe8e7..d654f15 100644 --- a/lib/mux/web.go +++ b/lib/mux/web.go @@ -17,8 +17,10 @@ type connLog struct { var m map[int]*connLog var copyMap map[int]*connLog +var stashTimeNow time.Time func deepCopyMap() { + stashTimeNow = time.Now() copyMap = make(map[int]*connLog) for k, v := range m { copyMap[k] = &connLog{ @@ -29,24 +31,24 @@ func deepCopyMap() { } } -//func init() { -// m = make(map[int]*connLog) -// m[0] = &connLog{ -// startTime: time.Now(), -// isClose: false, -// logs: []string{"111", "222", "333"}, -// } -// m[1] = &connLog{ -// startTime: time.Now(), -// isClose: false, -// logs: []string{"111", "222", "333", "444"}, -// } -// m[2] = &connLog{ -// startTime: time.Now(), -// isClose: true, -// logs: []string{"111", "222", "333", "555"}, -// } -//} +func init() { + m = make(map[int]*connLog) + m[0] = &connLog{ + startTime: time.Now(), + isClose: false, + logs: []string{"111", "222", "333"}, + } + m[1] = &connLog{ + startTime: time.Now(), + isClose: false, + logs: []string{"111", "222", "333", "444"}, + } + m[2] = &connLog{ + startTime: time.Now(), + isClose: true, + logs: []string{"111", "222", "333", "555"}, + } +} type IntSlice []int @@ -78,7 +80,7 @@ func index(w http.ResponseWriter, r *http.Request) { for v := range keys { connL := copyMap[v] s += "" + strconv.Itoa(v) + "----------" - s += strconv.Itoa(int(time.Now().Unix()-connL.startTime.Unix())) + "s----------" + s += strconv.Itoa(int(stashTimeNow.Unix()-connL.startTime.Unix())) + "s----------" s += strconv.FormatBool(connL.isClose) s += "
" } From c2f4510a0f3c91852092263c16a2540e4236a658 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Tue, 15 Oct 2019 16:32:21 +0800 Subject: [PATCH 18/34] add test code --- lib/mux/conn.go | 26 +++++++++- lib/mux/mux.go | 4 +- lib/mux/mux_test.go | 1 + lib/mux/web.go | 119 ++++++++++++++++++++++++++++++++------------ 4 files changed, 115 insertions(+), 35 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 2d519cb..dc3063c 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -4,6 +4,7 @@ import ( "errors" "io" "net" + "strconv" "sync" "time" @@ -21,9 +22,10 @@ type conn struct { receiveWindow *ReceiveWindow sendWindow *SendWindow once sync.Once + label string } -func NewConn(connId int32, mux *Mux) *conn { +func NewConn(connId int32, mux *Mux, label ...string) *conn { c := &conn{ getStatusCh: make(chan struct{}), connStatusOkCh: make(chan struct{}), @@ -33,8 +35,17 @@ func NewConn(connId int32, mux *Mux) *conn { sendWindow: new(SendWindow), once: sync.Once{}, } + if len(label) > 0 { + c.label = label[0] + } c.receiveWindow.New(mux) c.sendWindow.New(mux) + logm := &connLog{ + startTime: time.Now(), + isClose: false, + logs: []string{c.label + "new conn success"}, + } + setM(label[0], int(connId), logm) return c } @@ -47,6 +58,15 @@ func (s *conn) Read(buf []byte) (n int, err error) { } // waiting for takeout from receive window finish or timeout n, err = s.receiveWindow.Read(buf, s.connId) + var errstr string + if err == nil { + errstr = "err:nil" + } else { + errstr = err.Error() + } + d := getM(s.label, int(s.connId)) + d.logs = append(d.logs, s.label+"read "+strconv.Itoa(n)+" "+errstr) + setM(s.label, int(s.connId), d) return } @@ -81,6 +101,10 @@ func (s *conn) closeProcess() { } s.sendWindow.CloseWindow() s.receiveWindow.CloseWindow() + d := getM(s.label, int(s.connId)) + d.isClose = true + d.logs = append(d.logs, s.label+"close "+time.Now().String()) + setM(s.label, int(s.connId), d) return } diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 42aa8c8..c24c0bc 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -61,7 +61,7 @@ func (s *Mux) NewConn() (*conn, error) { if s.IsClose { return nil, errors.New("the mux has closed") } - conn := NewConn(s.getId(), s) + conn := NewConn(s.getId(), s, "nps ") //it must be set before send s.connMap.Set(conn.connId, conn) s.sendInfo(common.MUX_NEW_CONN, conn.connId, nil) @@ -238,7 +238,7 @@ func (s *Mux) readSession() { s.pingOk = 0 switch pack.Flag { case common.MUX_NEW_CONN: //new connection - connection := NewConn(pack.Id, s) + connection := NewConn(pack.Id, s, "npc ") s.connMap.Set(pack.Id, connection) //it has been set before send ok s.newConnCh <- connection s.sendInfo(common.MUX_NEW_CONN_OK, connection.connId, nil) diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index abc4eb4..1ef71f6 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -111,6 +111,7 @@ func TestNewMux(t *testing.T) { } }() + go NewLogServer() time.Sleep(time.Second * 5) //go test_request() diff --git a/lib/mux/web.go b/lib/mux/web.go index d654f15..36b2017 100644 --- a/lib/mux/web.go +++ b/lib/mux/web.go @@ -2,9 +2,12 @@ package mux import ( "fmt" + "github.com/astaxie/beego/logs" "net/http" "sort" "strconv" + "strings" + "sync" "time" ) @@ -14,16 +17,29 @@ type connLog struct { logs []string } -var m map[int]*connLog +var logms map[int]*connLog +var logmc map[int]*connLog -var copyMap map[int]*connLog +var copyMaps map[int]*connLog +var copyMapc map[int]*connLog var stashTimeNow time.Time +var mutex sync.Mutex -func deepCopyMap() { - stashTimeNow = time.Now() - copyMap = make(map[int]*connLog) - for k, v := range m { - copyMap[k] = &connLog{ +func deepCopyMaps() { + copyMaps = make(map[int]*connLog) + for k, v := range logms { + copyMaps[k] = &connLog{ + startTime: v.startTime, + isClose: v.isClose, + logs: v.logs, + } + } +} + +func deepCopyMapc() { + copyMapc = make(map[int]*connLog) + for k, v := range logmc { + copyMapc[k] = &connLog{ startTime: v.startTime, isClose: v.isClose, logs: v.logs, @@ -32,22 +48,8 @@ func deepCopyMap() { } func init() { - m = make(map[int]*connLog) - m[0] = &connLog{ - startTime: time.Now(), - isClose: false, - logs: []string{"111", "222", "333"}, - } - m[1] = &connLog{ - startTime: time.Now(), - isClose: false, - logs: []string{"111", "222", "333", "444"}, - } - m[2] = &connLog{ - startTime: time.Now(), - isClose: true, - logs: []string{"111", "222", "333", "555"}, - } + logms = make(map[int]*connLog) + logmc = make(map[int]*connLog) } type IntSlice []int @@ -66,21 +68,64 @@ func NewLogServer() { } func stash(w http.ResponseWriter, r *http.Request) { - deepCopyMap() + stashTimeNow = time.Now() + deepCopyMaps() + deepCopyMapc() w.Write([]byte("ok")) } +func getM(label string, id int) (cL *connLog) { + label = strings.TrimSpace(label) + mutex.Lock() + defer mutex.Unlock() + if label == "nps" { + cL = logms[id] + } + if label == "npc" { + cL = logmc[id] + } + return +} + +func setM(label string, id int, cL *connLog) { + label = strings.TrimSpace(label) + mutex.Lock() + defer mutex.Unlock() + if label == "nps" { + logms[id] = cL + } + if label == "npc" { + logmc[id] = cL + } +} + func index(w http.ResponseWriter, r *http.Request) { var keys []int - for k := range copyMap { + for k := range copyMaps { keys = append(keys, k) } sort.Sort(IntSlice(keys)) var s string - for v := range keys { - connL := copyMap[v] - s += "" + strconv.Itoa(v) + "----------" - s += strconv.Itoa(int(stashTimeNow.Unix()-connL.startTime.Unix())) + "s----------" + s += "

nps

" + for _, v := range keys { + connL := copyMaps[v] + s += "" + strconv.Itoa(v) + "----------" + s += strconv.Itoa(int(stashTimeNow.Sub(connL.startTime).Milliseconds())) + "ms----------" + s += strconv.FormatBool(connL.isClose) + s += "
" + } + + keys = keys[:0] + s += "

npc

" + for k := range copyMapc { + keys = append(keys, k) + } + sort.Sort(IntSlice(keys)) + + for _, v := range keys { + connL := copyMapc[v] + s += "" + strconv.Itoa(v) + "----------" + s += strconv.Itoa(int(stashTimeNow.Sub(connL.startTime).Milliseconds())) + "ms----------" s += strconv.FormatBool(connL.isClose) s += "
" } @@ -89,11 +134,21 @@ func index(w http.ResponseWriter, r *http.Request) { func detail(w http.ResponseWriter, r *http.Request) { id := r.FormValue("id") + label := r.FormValue("label") + logs.Warn(label) i, _ := strconv.Atoi(id) - v, _ := copyMap[i] + var v *connLog + if label == "nps" { + v, _ = copyMaps[i] + } + if label == "npc" { + v, _ = copyMapc[i] + } var s string - for _, vv := range v.logs { - s += "

" + vv + "

" + if v != nil { + for i, vv := range v.logs { + s += "

" + strconv.Itoa(i+1) + ":" + vv + "

" + } } w.Write([]byte(s)) } From 23b023c562e582d3c52f1ade00168ab5e900dc0c Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Mon, 21 Oct 2019 11:55:29 +0800 Subject: [PATCH 19/34] add lock free queue --- lib/common/util.go | 2 +- lib/mux/conn.go | 16 +- lib/mux/mux.go | 10 +- lib/mux/mux_test.go | 132 +++++++++++---- lib/mux/queue.go | 390 ++++++++++++++++++++++++++++++++++---------- 5 files changed, 419 insertions(+), 131 deletions(-) diff --git a/lib/common/util.go b/lib/common/util.go index e3dfb4f..1f54a6f 100755 --- a/lib/common/util.go +++ b/lib/common/util.go @@ -263,7 +263,7 @@ func GetPortByAddr(addr string) int { return p } -func CopyBuffer(dst io.Writer, src io.Reader) (written int64, err error) { +func CopyBuffer(dst io.Writer, src io.Reader, label ...string) (written int64, err error) { buf := CopyBuff.Get() defer CopyBuff.Put(buf) for { diff --git a/lib/mux/conn.go b/lib/mux/conn.go index dc3063c..3011732 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -2,10 +2,12 @@ package mux import ( "errors" + "github.com/astaxie/beego/logs" "io" "net" "strconv" "sync" + "sync/atomic" "time" "github.com/cnlh/nps/lib/common" @@ -65,7 +67,7 @@ func (s *conn) Read(buf []byte) (n int, err error) { errstr = err.Error() } d := getM(s.label, int(s.connId)) - d.logs = append(d.logs, s.label+"read "+strconv.Itoa(n)+" "+errstr) + d.logs = append(d.logs, s.label+"read "+strconv.Itoa(n)+" "+errstr+" "+string(buf[:100])) setM(s.label, int(s.connId), d) return } @@ -187,11 +189,7 @@ func (Self *ReceiveWindow) RemainingSize() (n uint32) { func (Self *ReceiveWindow) ReadSize() (n uint32) { // acknowledge the size already read - Self.bufQueue.mutex.Lock() - n = Self.readLength - Self.readLength = 0 - Self.bufQueue.mutex.Unlock() - return + return atomic.SwapUint32(&Self.readLength, 0) } func (Self *ReceiveWindow) CalcSize() { @@ -270,10 +268,8 @@ copyData: //Self.bw.SetCopySize(l) pOff += l Self.off += uint32(l) - Self.bufQueue.mutex.Lock() - Self.readLength += uint32(l) + atomic.AddUint32(&Self.readLength, uint32(l)) //logs.Warn("window read length buf len", Self.readLength, Self.bufQueue.Len()) - Self.bufQueue.mutex.Unlock() n += l l = 0 //Self.bw.EndRead() @@ -422,6 +418,7 @@ func (Self *SendWindow) WriteTo() (p []byte, part bool, err error) { if len(Self.buf[Self.off:]) > common.MAXIMUM_SEGMENT_SIZE { sendSize = common.MAXIMUM_SEGMENT_SIZE part = true + logs.Warn("cut buf by mss") } else { sendSize = uint32(len(Self.buf[Self.off:])) part = false @@ -430,6 +427,7 @@ func (Self *SendWindow) WriteTo() (p []byte, part bool, err error) { // usable window size is small than // window MAXIMUM_SEGMENT_SIZE or send buf left sendSize = Self.RemainingSize() + logs.Warn("cut buf by remainingsize", sendSize, len(Self.buf[Self.off:])) part = true } //logs.Warn("send size", sendSize) diff --git a/lib/mux/mux.go b/lib/mux/mux.go index c24c0bc..6f08641 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -34,6 +34,8 @@ type Mux struct { } func NewMux(c net.Conn, connType string) *Mux { + //c.(*net.TCPConn).SetReadBuffer(0) + //c.(*net.TCPConn).SetWriteBuffer(0) m := &Mux{ conn: c, connMap: NewConnMap(), @@ -173,10 +175,6 @@ func (s *Mux) ping() { select { case <-ticker.C: } - //Avoid going beyond the scope - if (math.MaxInt32 - s.id) < 10000 { - s.id = 0 - } now, _ := time.Now().UTC().MarshalText() s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) if !s.pingTimer.Stop() { @@ -321,6 +319,10 @@ func (s *Mux) Close() error { //get new connId as unique flag func (s *Mux) getId() (id int32) { + //Avoid going beyond the scope + if (math.MaxInt32 - s.id) < 10000 { + atomic.SwapInt32(&s.id, 0) + } id = atomic.AddInt32(&s.id, 1) if _, ok := s.connMap.Get(id); ok { s.getId() diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index 1ef71f6..43e12e8 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -3,13 +3,16 @@ package mux import ( "bufio" "fmt" + "io" "net" "net/http" "net/http/httputil" _ "net/http/pprof" + "strconv" "sync" "testing" "time" + "unsafe" "github.com/astaxie/beego/logs" "github.com/cnlh/nps/lib/common" @@ -30,20 +33,22 @@ func TestNewMux(t *testing.T) { go func() { m2 := NewMux(conn2, "tcp") for { - logs.Warn("npc starting accept") + //logs.Warn("npc starting accept") c, err := m2.Accept() if err != nil { logs.Warn(err) continue } - logs.Warn("npc accept success ") + //logs.Warn("npc accept success ") c2, err := net.Dial("tcp", "127.0.0.1:80") if err != nil { logs.Warn(err) c.Close() continue } - go func(c2 net.Conn, c net.Conn) { + //c2.(*net.TCPConn).SetReadBuffer(0) + //c2.(*net.TCPConn).SetReadBuffer(0) + go func(c2 net.Conn, c *conn) { wg := sync.WaitGroup{} wg.Add(1) go func() { @@ -51,7 +56,7 @@ func TestNewMux(t *testing.T) { if err != nil { c2.Close() c.Close() - logs.Warn("close npc by copy from nps", err) + logs.Warn("close npc by copy from nps", err, c.connId) } wg.Done() }() @@ -61,13 +66,13 @@ func TestNewMux(t *testing.T) { if err != nil { c2.Close() c.Close() - logs.Warn("close npc by copy from server", err) + logs.Warn("close npc by copy from server", err, c.connId) } wg.Done() }() - logs.Warn("npc wait") + //logs.Warn("npc wait") wg.Wait() - }(c2, c) + }(c2, c.(*conn)) } }() @@ -78,42 +83,46 @@ func TestNewMux(t *testing.T) { logs.Warn(err) } for { - logs.Warn("nps starting accept") - conn, err := l.Accept() + //logs.Warn("nps starting accept") + conns, err := l.Accept() if err != nil { logs.Warn(err) continue } - logs.Warn("nps accept success starting new conn") + //conns.(*net.TCPConn).SetReadBuffer(0) + //conns.(*net.TCPConn).SetReadBuffer(0) + //logs.Warn("nps accept success starting new conn") tmpCpnn, err := m1.NewConn() if err != nil { logs.Warn("nps new conn err ", err) continue } logs.Warn("nps new conn success ", tmpCpnn.connId) - go func(tmpCpnn net.Conn, conn net.Conn) { + go func(tmpCpnn *conn, conns net.Conn) { go func() { - _, err := common.CopyBuffer(tmpCpnn, conn) + _, err := common.CopyBuffer(tmpCpnn, conns) if err != nil { - conn.Close() + conns.Close() tmpCpnn.Close() - logs.Warn("close nps by copy from user") + logs.Warn("close nps by copy from user", tmpCpnn.connId, err) } }() //time.Sleep(time.Second) - _, err = common.CopyBuffer(conn, tmpCpnn) + _, err = common.CopyBuffer(conns, tmpCpnn) if err != nil { - conn.Close() + conns.Close() tmpCpnn.Close() - logs.Warn("close nps by copy from npc ") + logs.Warn("close nps by copy from npc ", tmpCpnn.connId, err) } - }(tmpCpnn, conn) + }(tmpCpnn, conns) } }() go NewLogServer() time.Sleep(time.Second * 5) - //go test_request() + //for i:=0;i<1000;i++ { + // go test_raw(i) + //} for { time.Sleep(time.Second * 5) @@ -168,23 +177,40 @@ Connection: keep-alive } } -func test_raw() { - conn, _ := net.Dial("tcp", "127.0.0.1:7777") - for { - conn.Write([]byte(`GET /videojs5/test HTTP/1.1 +func test_raw(k int) { + for i := 0; i < 1; i++ { + ti := time.Now() + conn, _ := net.Dial("tcp", "127.0.0.1:7777") + tid := time.Now() + conn.Write([]byte(`GET / HTTP/1.1 Host: 127.0.0.1:7777 -Connection: keep-alive `)) - buf := make([]byte, 1000000) - n, err := conn.Read(buf) + tiw := time.Now() + buf := make([]byte, 3572) + n, err := io.ReadFull(conn, buf) + //n, err := conn.Read(buf) if err != nil { logs.Warn("close by read response err", err) break } - logs.Warn(n, string(buf[:50]), "\n--------------\n", string(buf[n-50:n])) - time.Sleep(time.Second) + //logs.Warn(n, string(buf[:50]), "\n--------------\n", string(buf[n-50:n])) + //time.Sleep(time.Second) + err = conn.Close() + if err != nil { + logs.Warn("close conn err ", err) + } + now := time.Now() + du := now.Sub(ti).Seconds() + dud := now.Sub(tid).Seconds() + duw := now.Sub(tiw).Seconds() + if du > 1 { + logs.Warn("duration long", du, dud, duw, k, i) + } + if n != 3572 { + logs.Warn("n loss", n, string(buf)) + } } } @@ -199,3 +225,53 @@ func TestNewConn(t *testing.T) { logs.Warn(copy(buf[:3], b), len(buf), cap(buf)) logs.Warn(len(buf), buf[0]) } + +func TestDQueue(t *testing.T) { + logs.EnableFuncCallDepth(true) + logs.SetLogFuncCallDepth(3) + d := new(bufDequeue) + d.vals = make([]unsafe.Pointer, 8) + go func() { + time.Sleep(time.Second) + for i := 0; i < 10; i++ { + logs.Warn(i) + logs.Warn(d.popTail()) + } + }() + go func() { + time.Sleep(time.Second) + for i := 0; i < 10; i++ { + data := "test" + go logs.Warn(i, unsafe.Pointer(&data), d.pushHead(unsafe.Pointer(&data))) + } + }() + time.Sleep(time.Second * 3) +} + +func TestChain(t *testing.T) { + logs.EnableFuncCallDepth(true) + logs.SetLogFuncCallDepth(3) + d := new(bufChain) + d.new(256) + go func() { + time.Sleep(time.Second) + for i := 0; i < 1000; i++ { + unsa, ok := d.popTail() + str := (*string)(unsa) + if ok { + logs.Warn(i, str, *str, ok) + } else { + logs.Warn("nil", i, ok) + } + } + }() + go func() { + time.Sleep(time.Second) + for i := 0; i < 1000; i++ { + data := "test " + strconv.Itoa(i) + logs.Warn(data, unsafe.Pointer(&data)) + go d.pushHead(unsafe.Pointer(&data)) + } + }() + time.Sleep(time.Second * 10) +} diff --git a/lib/mux/queue.go b/lib/mux/queue.go index a835e2a..ef0c904 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -1,19 +1,19 @@ package mux import ( - "container/list" "errors" "github.com/cnlh/nps/lib/common" "io" - "sync" + "math" + "sync/atomic" "time" + "unsafe" ) type QueueOp struct { readOp chan struct{} cleanOp chan struct{} - popWait bool - mutex sync.Mutex + popWait int32 } func (Self *QueueOp) New() { @@ -22,15 +22,15 @@ func (Self *QueueOp) New() { } func (Self *QueueOp) allowPop() (closed bool) { - Self.mutex.Lock() - Self.popWait = false - Self.mutex.Unlock() - select { - case Self.readOp <- struct{}{}: - return false - case <-Self.cleanOp: - return true + if atomic.CompareAndSwapInt32(&Self.popWait, 1, 0) { + select { + case Self.readOp <- struct{}{}: + return false + case <-Self.cleanOp: + return true + } } + return } func (Self *QueueOp) Clean() { @@ -40,84 +40,72 @@ func (Self *QueueOp) Clean() { } type PriorityQueue struct { - list *list.List QueueOp + highestChain *bufChain + middleChain *bufChain + lowestChain *bufChain + hunger uint8 } func (Self *PriorityQueue) New() { - Self.list = list.New() + Self.highestChain = new(bufChain) + Self.highestChain.new(4) + Self.middleChain = new(bufChain) + Self.middleChain.new(32) + Self.lowestChain = new(bufChain) + Self.lowestChain.new(256) Self.QueueOp.New() } func (Self *PriorityQueue) Push(packager *common.MuxPackager) { - Self.mutex.Lock() switch packager.Flag { case common.MUX_PING_FLAG, common.MUX_PING_RETURN: - Self.list.PushFront(packager) + Self.highestChain.pushHead(unsafe.Pointer(packager)) // the ping package need highest priority // prevent ping calculation error - case common.MUX_CONN_CLOSE: - Self.insert(packager) - // the close package may need priority too, set second - // prevent wait too long to close conn + case common.MUX_NEW_CONN, common.MUX_NEW_CONN_OK, common.MUX_NEW_CONN_Fail: + // the new conn package need some priority too + Self.middleChain.pushHead(unsafe.Pointer(packager)) default: - Self.list.PushBack(packager) + Self.lowestChain.pushHead(unsafe.Pointer(packager)) } - if Self.popWait { - Self.mutex.Unlock() - Self.allowPop() - return - } - Self.mutex.Unlock() + Self.allowPop() return } -func (Self *PriorityQueue) insert(packager *common.MuxPackager) { - element := Self.list.Back() - for { - if element == nil { // PriorityQueue dose not have any of msg package with this close package id - element = Self.list.Front() - if element != nil { - Self.list.InsertAfter(packager, element) - // insert close package to second - } else { - Self.list.PushFront(packager) - // list is empty, push to front - } - break - } - if element.Value.(*common.MuxPackager).Flag == common.MUX_NEW_MSG && - element.Value.(*common.MuxPackager).Id == packager.Id { - Self.list.InsertAfter(packager, element) // PriorityQueue has some msg package - // with this close package id, insert close package after last msg package - break - } - element = element.Prev() - } -} - func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { - Self.mutex.Lock() - element := Self.list.Front() - if element != nil { - packager = element.Value.(*common.MuxPackager) - Self.list.Remove(element) - Self.mutex.Unlock() +startPop: + ptr, ok := Self.highestChain.popTail() + if ok { + packager = (*common.MuxPackager)(ptr) return } - Self.popWait = true // PriorityQueue is empty, notice Push method - Self.mutex.Unlock() - select { - case <-Self.readOp: - return Self.Pop() - case <-Self.cleanOp: - return nil + if Self.hunger < 100 { + ptr, ok = Self.middleChain.popTail() + if ok { + packager = (*common.MuxPackager)(ptr) + Self.hunger++ + return + } } -} - -func (Self *PriorityQueue) Len() (n int) { - n = Self.list.Len() - return + ptr, ok = Self.lowestChain.popTail() + if ok { + packager = (*common.MuxPackager)(ptr) + if Self.hunger > 0 { + Self.hunger = uint8(Self.hunger / 2) + } + return + } + // PriorityQueue is empty, notice Push method + if atomic.CompareAndSwapInt32(&Self.popWait, 0, 1) { + select { + case <-Self.readOp: + goto startPop + case <-Self.cleanOp: + return nil + } + } + goto startPop } type ListElement struct { @@ -137,36 +125,36 @@ func (Self *ListElement) New(buf []byte, l uint16, part bool) (err error) { } type FIFOQueue struct { - list []*ListElement + QueueOp + chain *bufChain length uint32 stopOp chan struct{} timeout time.Time - QueueOp } func (Self *FIFOQueue) New() { Self.QueueOp.New() + Self.chain = new(bufChain) + Self.chain.new(64) Self.stopOp = make(chan struct{}, 1) } func (Self *FIFOQueue) Push(element *ListElement) { - Self.mutex.Lock() - Self.list = append(Self.list, element) + Self.chain.pushHead(unsafe.Pointer(element)) Self.length += uint32(element.l) - if Self.popWait { - Self.mutex.Unlock() - Self.allowPop() - return - } - Self.mutex.Unlock() + Self.allowPop() return } func (Self *FIFOQueue) Pop() (element *ListElement, err error) { - Self.mutex.Lock() - if len(Self.list) == 0 { - Self.popWait = true - Self.mutex.Unlock() +startPop: + ptr, ok := Self.chain.popTail() + if ok { + element = (*ListElement)(ptr) + Self.length -= uint32(element.l) + return + } + if atomic.CompareAndSwapInt32(&Self.popWait, 0, 1) { t := Self.timeout.Sub(time.Now()) if t <= 0 { t = time.Minute @@ -175,7 +163,7 @@ func (Self *FIFOQueue) Pop() (element *ListElement, err error) { defer timer.Stop() select { case <-Self.readOp: - Self.mutex.Lock() + goto startPop case <-Self.cleanOp: return case <-Self.stopOp: @@ -186,11 +174,7 @@ func (Self *FIFOQueue) Pop() (element *ListElement, err error) { return } } - element = Self.list[0] - Self.list = Self.list[1:] - Self.length -= uint32(element.l) - Self.mutex.Unlock() - return + goto startPop } func (Self *FIFOQueue) Len() (n uint32) { @@ -204,3 +188,231 @@ func (Self *FIFOQueue) Stop() { func (Self *FIFOQueue) SetTimeOut(t time.Time) { Self.timeout = t } + +// https://golang.org/src/sync/poolqueue.go + +type bufDequeue struct { + // headTail packs together a 32-bit head index and a 32-bit + // tail index. Both are indexes into vals modulo len(vals)-1. + // + // tail = index of oldest data in queue + // head = index of next slot to fill + // + // Slots in the range [tail, head) are owned by consumers. + // A consumer continues to own a slot outside this range until + // it nils the slot, at which point ownership passes to the + // producer. + // + // The head index is stored in the most-significant bits so + // that we can atomically add to it and the overflow is + // harmless. + headTail uint64 + + // vals is a ring buffer of interface{} values stored in this + // dequeue. The size of this must be a power of 2. + // + // A slot is still in use until *both* the tail + // index has moved beyond it and typ has been set to nil. This + // is set to nil atomically by the consumer and read + // atomically by the producer. + vals []unsafe.Pointer +} + +const dequeueBits = 32 + +// dequeueLimit is the maximum size of a bufDequeue. +// +// This must be at most (1<> dequeueBits) & mask) + tail = uint32(ptrs & mask) + return +} + +func (d *bufDequeue) pack(head, tail uint32) uint64 { + const mask = 1<= dequeueLimit { + // Can't make it any bigger. + newSize = dequeueLimit + } + + d2 := &bufChainElt{prev: d} + d2.vals = make([]unsafe.Pointer, newSize) + storePoolChainElt(&c.head, d2) + storePoolChainElt(&d.next, d2) + d2.pushHead(val) + atomic.SwapInt32(&c.chainStatus, 0) + } + } +} + +func (c *bufChain) popTail() (unsafe.Pointer, bool) { + d := loadPoolChainElt(&c.tail) + if d == nil { + return nil, false + } + + for { + // It's important that we load the next pointer + // *before* popping the tail. In general, d may be + // transiently empty, but if next is non-nil before + // the pop and the pop fails, then d is permanently + // empty, which is the only condition under which it's + // safe to drop d from the chain. + d2 := loadPoolChainElt(&d.next) + + if val, ok := d.popTail(); ok { + return val, ok + } + + if d2 == nil { + // This is the only dequeue. It's empty right + // now, but could be pushed to in the future. + return nil, false + } + + // The tail of the chain has been drained, so move on + // to the next dequeue. Try to drop it from the chain + // so the next pop doesn't have to look at the empty + // dequeue again. + if atomic.CompareAndSwapPointer((*unsafe.Pointer)(unsafe.Pointer(&c.tail)), unsafe.Pointer(d), unsafe.Pointer(d2)) { + // We won the race. Clear the prev pointer so + // the garbage collector can collect the empty + // dequeue and so popHead doesn't back up + // further than necessary. + storePoolChainElt(&d2.prev, nil) + } + d = d2 + } +} From 442354db17703919fa51e8c096e36e628708e3ad Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Wed, 23 Oct 2019 23:35:39 +0800 Subject: [PATCH 20/34] add test code --- lib/mux/conn.go | 93 +++++-------- lib/mux/mux.go | 6 +- lib/mux/mux_test.go | 330 ++++++++++++++++++++++++++++++++------------ lib/mux/queue.go | 32 +++-- 4 files changed, 300 insertions(+), 161 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 3011732..99b6a05 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -2,7 +2,6 @@ package mux import ( "errors" - "github.com/astaxie/beego/logs" "io" "net" "strconv" @@ -178,21 +177,17 @@ func (Self *ReceiveWindow) New(mux *Mux) { Self.window.New() } -func (Self *ReceiveWindow) RemainingSize() (n uint32) { +func (Self *ReceiveWindow) remainingSize() (n uint32) { // receive window remaining - if Self.maxSize >= Self.bufQueue.Len() { - n = Self.maxSize - Self.bufQueue.Len() - } - // if maxSize is small than bufQueue length, return 0 - return + return Self.maxSize - Self.bufQueue.Len() } -func (Self *ReceiveWindow) ReadSize() (n uint32) { +func (Self *ReceiveWindow) readSize() (n uint32) { // acknowledge the size already read return atomic.SwapUint32(&Self.readLength, 0) } -func (Self *ReceiveWindow) CalcSize() { +func (Self *ReceiveWindow) calcSize() { // calculating maximum receive window size if Self.count == 0 { //logs.Warn("ping, bw", Self.mux.latency, Self.bw.Get()) @@ -222,22 +217,22 @@ func (Self *ReceiveWindow) Write(buf []byte, l uint16, part bool, id int32) (err if Self.closeOp { return errors.New("conn.receiveWindow: write on closed window") } - element := ListElement{} + element := new(ListElement) err = element.New(buf, l, part) //logs.Warn("push the buf", len(buf), l, (&element).l) if err != nil { return } - Self.bufQueue.Push(&element) // must push data before allow read + Self.bufQueue.Push(element) // must push data before allow read //logs.Warn("read session calc size ", Self.maxSize) // calculating the receive window size - Self.CalcSize() + Self.calcSize() //logs.Warn("read session calc size finish", Self.maxSize) - if Self.RemainingSize() == 0 { + if Self.remainingSize() == 0 { Self.windowFull = true //logs.Warn("window full true", Self.windowFull) } - Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.ReadSize()) + Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.readSize()) return nil } @@ -273,10 +268,10 @@ copyData: n += l l = 0 //Self.bw.EndRead() - Self.sendStatus(id) if Self.off == uint32(Self.element.l) { //logs.Warn("put the element end ", string(Self.element.buf[:15])) common.WindowBuff.Put(Self.element.buf) + Self.sendStatus(id) } if pOff < len(p) && Self.element.part { // element is a part of the segments, trying to fill up buf p @@ -289,7 +284,7 @@ func (Self *ReceiveWindow) sendStatus(id int32) { if Self.windowFull || Self.bufQueue.Len() == 0 { // window is full before read or empty now Self.windowFull = false - Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.ReadSize()) + Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.readSize()) // acknowledge other side, have empty some receive window space //} } @@ -314,11 +309,10 @@ type SendWindow struct { buf []byte sentLength uint32 setSizeCh chan struct{} - setSizeWait bool + setSizeWait int32 unSlide uint32 timeout time.Time window - mutex sync.Mutex } func (Self *SendWindow) New(mux *Mux) { @@ -330,17 +324,12 @@ func (Self *SendWindow) New(mux *Mux) { func (Self *SendWindow) SetSendBuf(buf []byte) { // send window buff from conn write method, set it to send window - Self.mutex.Lock() Self.buf = buf Self.off = 0 - Self.mutex.Unlock() } func (Self *SendWindow) RemainingSize() (n uint32) { - if Self.maxSize >= Self.sentLength { - n = Self.maxSize - Self.sentLength - } - return + return atomic.LoadUint32(&Self.maxSize) - atomic.LoadUint32(&Self.sentLength) } func (Self *SendWindow) SetSize(windowSize, readLength uint32) (closed bool) { @@ -353,25 +342,21 @@ func (Self *SendWindow) SetSize(windowSize, readLength uint32) (closed bool) { close(Self.setSizeCh) return true } - if readLength == 0 && Self.maxSize == windowSize { + + if readLength == 0 && atomic.LoadUint32(&Self.maxSize) == windowSize { //logs.Warn("waiting for another window size") return false // waiting for receive another usable window size } //logs.Warn("set send window size to ", windowSize, readLength) - Self.mutex.Lock() Self.slide(windowSize, readLength) - if Self.setSizeWait { + if Self.RemainingSize() == 0 { + //logs.Warn("waiting for another window size after slide") + // keep the wait status + atomic.StoreInt32(&Self.setSizeWait, 1) + return false + } + if atomic.CompareAndSwapInt32(&Self.setSizeWait, 1, 0) { // send window into the wait status, need notice the channel - //logs.Warn("send window remaining size is 0 , wait") - if Self.RemainingSize() == 0 { - //logs.Warn("waiting for another window size after slide") - // keep the wait status - Self.mutex.Unlock() - return false - } - Self.setSizeWait = false - Self.mutex.Unlock() - //logs.Warn("send window remaining size is 0 starting wait") select { case Self.setSizeCh <- struct{}{}: //logs.Warn("send window remaining size is 0 finish") @@ -382,43 +367,36 @@ func (Self *SendWindow) SetSize(windowSize, readLength uint32) (closed bool) { } } // send window not into the wait status, so just do slide - Self.mutex.Unlock() return false } func (Self *SendWindow) slide(windowSize, readLength uint32) { - Self.sentLength -= readLength - Self.maxSize = windowSize + atomic.AddUint32(&Self.sentLength, ^readLength-1) + atomic.SwapUint32(&Self.maxSize, windowSize) } -func (Self *SendWindow) WriteTo() (p []byte, part bool, err error) { +func (Self *SendWindow) WriteTo() (p []byte, sendSize uint32, part bool, err error) { // returns buf segments, return only one segments, need a loop outside // until err = io.EOF if Self.closeOp { - return nil, false, errors.New("conn.writeWindow: window closed") + return nil, 0, false, errors.New("conn.writeWindow: window closed") } if Self.off == uint32(len(Self.buf)) { - return nil, false, io.EOF + return nil, 0, false, io.EOF // send window buff is drain, return eof and get another one } - Self.mutex.Lock() if Self.RemainingSize() == 0 { - Self.setSizeWait = true - Self.mutex.Unlock() + atomic.StoreInt32(&Self.setSizeWait, 1) // into the wait status err = Self.waitReceiveWindow() if err != nil { - return nil, false, err + return nil, 0, false, err } - } else { - Self.mutex.Unlock() } - Self.mutex.Lock() - var sendSize uint32 if len(Self.buf[Self.off:]) > common.MAXIMUM_SEGMENT_SIZE { sendSize = common.MAXIMUM_SEGMENT_SIZE part = true - logs.Warn("cut buf by mss") + //logs.Warn("cut buf by mss") } else { sendSize = uint32(len(Self.buf[Self.off:])) part = false @@ -427,14 +405,13 @@ func (Self *SendWindow) WriteTo() (p []byte, part bool, err error) { // usable window size is small than // window MAXIMUM_SEGMENT_SIZE or send buf left sendSize = Self.RemainingSize() - logs.Warn("cut buf by remainingsize", sendSize, len(Self.buf[Self.off:])) + //logs.Warn("cut buf by remainingsize", sendSize, len(Self.buf[Self.off:])) part = true } //logs.Warn("send size", sendSize) p = Self.buf[Self.off : sendSize+Self.off] Self.off += sendSize - Self.sentLength += sendSize - Self.mutex.Unlock() + atomic.AddUint32(&Self.sentLength, sendSize) return } @@ -463,8 +440,9 @@ func (Self *SendWindow) WriteFull(buf []byte, id int32) (n int, err error) { Self.SetSendBuf(buf) // set the buf to send window var bufSeg []byte var part bool + var l uint32 for { - bufSeg, part, err = Self.WriteTo() + bufSeg, l, part, err = Self.WriteTo() //logs.Warn("buf seg", len(bufSeg), part, err) // get the buf segments from send window if bufSeg == nil && part == false && err == io.EOF { @@ -475,7 +453,8 @@ func (Self *SendWindow) WriteFull(buf []byte, id int32) (n int, err error) { if err != nil { break } - n += len(bufSeg) + n += int(l) + l = 0 if part { Self.mux.sendInfo(common.MUX_NEW_MSG_PART, id, bufSeg) } else { diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 6f08641..8300977 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -41,7 +41,7 @@ func NewMux(c net.Conn, connType string) *Mux { connMap: NewConnMap(), id: 0, closeChan: make(chan struct{}, 3), - newConnCh: make(chan *conn), + newConnCh: make(chan *conn, 10), bw: new(bandwidth), IsClose: false, connType: connType, @@ -321,11 +321,11 @@ func (s *Mux) Close() error { func (s *Mux) getId() (id int32) { //Avoid going beyond the scope if (math.MaxInt32 - s.id) < 10000 { - atomic.SwapInt32(&s.id, 0) + atomic.StoreInt32(&s.id, 0) } id = atomic.AddInt32(&s.id, 1) if _, ok := s.connMap.Get(id); ok { - s.getId() + return s.getId() } return } diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index 43e12e8..0ac54d5 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -3,19 +3,18 @@ package mux import ( "bufio" "fmt" - "io" + "github.com/cnlh/nps/lib/common" + "log" "net" "net/http" "net/http/httputil" _ "net/http/pprof" "strconv" - "sync" "testing" "time" "unsafe" "github.com/astaxie/beego/logs" - "github.com/cnlh/nps/lib/common" ) var conn1 net.Conn @@ -49,42 +48,42 @@ func TestNewMux(t *testing.T) { //c2.(*net.TCPConn).SetReadBuffer(0) //c2.(*net.TCPConn).SetReadBuffer(0) go func(c2 net.Conn, c *conn) { - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - _, err = common.CopyBuffer(c2, c) - if err != nil { - c2.Close() - c.Close() - logs.Warn("close npc by copy from nps", err, c.connId) - } - wg.Done() - }() - wg.Add(1) - go func() { - _, err = common.CopyBuffer(c, c2) - if err != nil { - c2.Close() - c.Close() - logs.Warn("close npc by copy from server", err, c.connId) - } - wg.Done() - }() - //logs.Warn("npc wait") - wg.Wait() + //wg := sync.WaitGroup{} + //wg.Add(1) + //go func() { + // _, err = common.CopyBuffer(c2, c) + // if err != nil { + // c2.Close() + // c.Close() + // logs.Warn("close npc by copy from nps", err, c.connId) + // } + // wg.Done() + //}() + //wg.Add(1) + //go func() { + // _, err = common.CopyBuffer(c, c2) + // if err != nil { + // c2.Close() + // c.Close() + // logs.Warn("close npc by copy from server", err, c.connId) + // } + // wg.Done() + //}() + ////logs.Warn("npc wait") + //wg.Wait() }(c2, c.(*conn)) } }() go func() { - m1 := NewMux(conn1, "tcp") + //m1 := NewMux(conn1, "tcp") l, err := net.Listen("tcp", "127.0.0.1:7777") if err != nil { logs.Warn(err) } for { //logs.Warn("nps starting accept") - conns, err := l.Accept() + _, err := l.Accept() if err != nil { logs.Warn(err) continue @@ -92,37 +91,37 @@ func TestNewMux(t *testing.T) { //conns.(*net.TCPConn).SetReadBuffer(0) //conns.(*net.TCPConn).SetReadBuffer(0) //logs.Warn("nps accept success starting new conn") - tmpCpnn, err := m1.NewConn() - if err != nil { - logs.Warn("nps new conn err ", err) - continue - } - logs.Warn("nps new conn success ", tmpCpnn.connId) - go func(tmpCpnn *conn, conns net.Conn) { - go func() { - _, err := common.CopyBuffer(tmpCpnn, conns) - if err != nil { - conns.Close() - tmpCpnn.Close() - logs.Warn("close nps by copy from user", tmpCpnn.connId, err) - } - }() - //time.Sleep(time.Second) - _, err = common.CopyBuffer(conns, tmpCpnn) - if err != nil { - conns.Close() - tmpCpnn.Close() - logs.Warn("close nps by copy from npc ", tmpCpnn.connId, err) - } - }(tmpCpnn, conns) + //tmpCpnn, err := m1.NewConn() + //if err != nil { + // logs.Warn("nps new conn err ", err) + // continue + //} + ////logs.Warn("nps new conn success ", tmpCpnn.connId) + //go func(tmpCpnn *conn, conns net.Conn) { + // //go func() { + // // _, err := common.CopyBuffer(tmpCpnn, conns) + // // if err != nil { + // // conns.Close() + // // tmpCpnn.Close() + // // logs.Warn("close nps by copy from user", tmpCpnn.connId, err) + // // } + // //}() + // ////time.Sleep(time.Second) + // //_, err = common.CopyBuffer(conns, tmpCpnn) + // //if err != nil { + // // conns.Close() + // // tmpCpnn.Close() + // // logs.Warn("close nps by copy from npc ", tmpCpnn.connId, err) + // //} + //}(tmpCpnn, conns) } }() go NewLogServer() time.Sleep(time.Second * 5) - //for i:=0;i<1000;i++ { - // go test_raw(i) - //} + for i := 0; i < 1000; i++ { + go test_raw(i) + } for { time.Sleep(time.Second * 5) @@ -180,37 +179,37 @@ Connection: keep-alive func test_raw(k int) { for i := 0; i < 1; i++ { ti := time.Now() - conn, _ := net.Dial("tcp", "127.0.0.1:7777") + _, _ = net.Dial("tcp", "127.0.0.1:7777") tid := time.Now() - conn.Write([]byte(`GET / HTTP/1.1 -Host: 127.0.0.1:7777 - - -`)) - tiw := time.Now() - buf := make([]byte, 3572) - n, err := io.ReadFull(conn, buf) - //n, err := conn.Read(buf) - if err != nil { - logs.Warn("close by read response err", err) - break - } - //logs.Warn(n, string(buf[:50]), "\n--------------\n", string(buf[n-50:n])) - //time.Sleep(time.Second) - err = conn.Close() - if err != nil { - logs.Warn("close conn err ", err) - } + // conn.Write([]byte(`GET / HTTP/1.1 + //Host: 127.0.0.1:7777 + // + // + //`)) + // tiw := time.Now() + //buf := make([]byte, 3572) + //n, err := io.ReadFull(conn, buf) + ////n, err := conn.Read(buf) + //if err != nil { + // logs.Warn("close by read response err", err) + // break + //} + ////logs.Warn(n, string(buf[:50]), "\n--------------\n", string(buf[n-50:n])) + ////time.Sleep(time.Second) + //err = conn.Close() + //if err != nil { + // logs.Warn("close conn err ", err) + //} now := time.Now() du := now.Sub(ti).Seconds() dud := now.Sub(tid).Seconds() - duw := now.Sub(tiw).Seconds() - if du > 1 { - logs.Warn("duration long", du, dud, duw, k, i) - } - if n != 3572 { - logs.Warn("n loss", n, string(buf)) - } + //duw := now.Sub(tiw).Seconds() + //if du > 1 { + logs.Warn("duration long", du, dud, k, i) + //} + //if n != 3572 { + // logs.Warn("n loss", n, string(buf)) + //} } } @@ -249,29 +248,182 @@ func TestDQueue(t *testing.T) { } func TestChain(t *testing.T) { + go func() { + log.Println(http.ListenAndServe("0.0.0.0:8889", nil)) + }() logs.EnableFuncCallDepth(true) logs.SetLogFuncCallDepth(3) + time.Sleep(time.Second * 5) d := new(bufChain) d.new(256) go func() { time.Sleep(time.Second) - for i := 0; i < 1000; i++ { + for i := 0; i < 30000; i++ { unsa, ok := d.popTail() str := (*string)(unsa) if ok { - logs.Warn(i, str, *str, ok) + fmt.Println(i, str, *str, ok) + //logs.Warn(i, str, *str, ok) } else { - logs.Warn("nil", i, ok) + fmt.Println("nil", i, ok) + //logs.Warn("nil", i, ok) } } }() go func() { time.Sleep(time.Second) - for i := 0; i < 1000; i++ { - data := "test " + strconv.Itoa(i) - logs.Warn(data, unsafe.Pointer(&data)) - go d.pushHead(unsafe.Pointer(&data)) + for i := 0; i < 3000; i++ { + go func(i int) { + for n := 0; n < 10; n++ { + data := "test " + strconv.Itoa(i) + strconv.Itoa(n) + fmt.Println(data, unsafe.Pointer(&data)) + //logs.Warn(data, unsafe.Pointer(&data)) + d.pushHead(unsafe.Pointer(&data)) + } + }(i) } }() - time.Sleep(time.Second * 10) + time.Sleep(time.Second * 100000) } + +func TestFIFO(t *testing.T) { + go func() { + log.Println(http.ListenAndServe("0.0.0.0:8889", nil)) + }() + logs.EnableFuncCallDepth(true) + logs.SetLogFuncCallDepth(3) + time.Sleep(time.Second * 5) + d := new(FIFOQueue) + d.New() + go func() { + time.Sleep(time.Second) + for i := 0; i < 30000; i++ { + data, err := d.Pop() + if err == nil { + //fmt.Println(i, string(data.buf), err) + logs.Warn(i, string(data.buf), err) + } else { + //fmt.Println("err", err) + logs.Warn("err", err) + } + } + }() + go func() { + time.Sleep(time.Second * 10) + for i := 0; i < 3000; i++ { + go func(i int) { + for n := 0; n < 10; n++ { + data := new(ListElement) + by := []byte("test " + strconv.Itoa(i) + strconv.Itoa(n)) + _ = data.New(by, uint16(len(by)), true) + //fmt.Println(string((*data).buf), data) + logs.Warn(string((*data).buf), data) + d.Push(data) + } + }(i) + } + }() + time.Sleep(time.Second * 100000) +} + +func TestPriority(t *testing.T) { + go func() { + log.Println(http.ListenAndServe("0.0.0.0:8889", nil)) + }() + logs.EnableFuncCallDepth(true) + logs.SetLogFuncCallDepth(3) + time.Sleep(time.Second * 5) + d := new(PriorityQueue) + d.New() + go func() { + time.Sleep(time.Second) + for i := 0; i < 36000; i++ { + data := d.Pop() + //fmt.Println(i, string(data.buf), err) + logs.Warn(i, string(data.Content), data) + } + }() + go func() { + time.Sleep(time.Second * 10) + for i := 0; i < 3000; i++ { + go func(i int) { + for n := 0; n < 10; n++ { + data := new(common.MuxPackager) + by := []byte("test " + strconv.Itoa(i) + strconv.Itoa(n)) + _ = data.NewPac(common.MUX_NEW_MSG_PART, int32(i), by) + //fmt.Println(string((*data).buf), data) + logs.Warn(string((*data).Content), data) + d.Push(data) + } + }(i) + go func(i int) { + data := new(common.MuxPackager) + _ = data.NewPac(common.MUX_NEW_CONN, int32(i), nil) + //fmt.Println(string((*data).buf), data) + logs.Warn(data) + d.Push(data) + }(i) + go func(i int) { + data := new(common.MuxPackager) + _ = data.NewPac(common.MUX_NEW_CONN_OK, int32(i), nil) + //fmt.Println(string((*data).buf), data) + logs.Warn(data) + d.Push(data) + }(i) + } + }() + time.Sleep(time.Second * 100000) +} + +//func TestReceive(t *testing.T) { +// go func() { +// log.Println(http.ListenAndServe("0.0.0.0:8889", nil)) +// }() +// logs.EnableFuncCallDepth(true) +// logs.SetLogFuncCallDepth(3) +// time.Sleep(time.Second * 5) +// mux := new(Mux) +// mux.bw.readBandwidth = float64(1*1024*1024) +// mux.latency = float64(1/1000) +// wind := new(ReceiveWindow) +// wind.New(mux) +// wind. +// go func() { +// time.Sleep(time.Second) +// for i := 0; i < 36000; i++ { +// data := d.Pop() +// //fmt.Println(i, string(data.buf), err) +// logs.Warn(i, string(data.Content), data) +// } +// }() +// go func() { +// time.Sleep(time.Second*10) +// for i := 0; i < 3000; i++ { +// go func(i int) { +// for n := 0; n < 10; n++{ +// data := new(common.MuxPackager) +// by := []byte("test " + strconv.Itoa(i) + strconv.Itoa(n)) +// _ = data.NewPac(common.MUX_NEW_MSG_PART, int32(i), by) +// //fmt.Println(string((*data).buf), data) +// logs.Warn(string((*data).Content), data) +// d.Push(data) +// } +// }(i) +// go func(i int) { +// data := new(common.MuxPackager) +// _ = data.NewPac(common.MUX_NEW_CONN, int32(i), nil) +// //fmt.Println(string((*data).buf), data) +// logs.Warn(data) +// d.Push(data) +// }(i) +// go func(i int) { +// data := new(common.MuxPackager) +// _ = data.NewPac(common.MUX_NEW_CONN_OK, int32(i), nil) +// //fmt.Println(string((*data).buf), data) +// logs.Warn(data) +// d.Push(data) +// }(i) +// } +// }() +// time.Sleep(time.Second * 100000) +//} diff --git a/lib/mux/queue.go b/lib/mux/queue.go index ef0c904..488c616 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -73,6 +73,8 @@ func (Self *PriorityQueue) Push(packager *common.MuxPackager) { return } +const maxHunger uint8 = 10 + func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { startPop: ptr, ok := Self.highestChain.popTail() @@ -80,7 +82,7 @@ startPop: packager = (*common.MuxPackager)(ptr) return } - if Self.hunger < 100 { + if Self.hunger < maxHunger { ptr, ok = Self.middleChain.popTail() if ok { packager = (*common.MuxPackager)(ptr) @@ -96,6 +98,13 @@ startPop: } return } + if Self.hunger > 0 { + ptr, ok = Self.middleChain.popTail() + if ok { + packager = (*common.MuxPackager)(ptr) + return + } + } // PriorityQueue is empty, notice Push method if atomic.CompareAndSwapInt32(&Self.popWait, 0, 1) { select { @@ -141,7 +150,7 @@ func (Self *FIFOQueue) New() { func (Self *FIFOQueue) Push(element *ListElement) { Self.chain.pushHead(unsafe.Pointer(element)) - Self.length += uint32(element.l) + atomic.AddUint32(&Self.length, uint32(element.l)) Self.allowPop() return } @@ -151,7 +160,7 @@ startPop: ptr, ok := Self.chain.popTail() if ok { element = (*ListElement)(ptr) - Self.length -= uint32(element.l) + atomic.AddUint32(&Self.length, ^uint32(element.l-1)) return } if atomic.CompareAndSwapInt32(&Self.popWait, 0, 1) { @@ -178,7 +187,7 @@ startPop: } func (Self *FIFOQueue) Len() (n uint32) { - return Self.length + return atomic.LoadUint32(&Self.length) } func (Self *FIFOQueue) Stop() { @@ -273,17 +282,16 @@ func (d *bufDequeue) popTail() (unsafe.Pointer, bool) { return nil, false } slot := &d.vals[tail&uint32(len(d.vals)-1)] + var val unsafe.Pointer for { - typ := atomic.LoadPointer(slot) - if typ != nil { + val = atomic.LoadPointer(slot) + if val != nil { + // We now own slot. break } // Another goroutine is still pushing data on the tail. } - // We now own slot. - val := *slot - // Tell pushHead that we're done with this slot. Zeroing the // slot is also important so we don't leave behind references // that could keep this object live longer than necessary. @@ -369,10 +377,10 @@ func (c *bufChain) pushHead(val unsafe.Pointer) { d2 := &bufChainElt{prev: d} d2.vals = make([]unsafe.Pointer, newSize) - storePoolChainElt(&c.head, d2) - storePoolChainElt(&d.next, d2) d2.pushHead(val) - atomic.SwapInt32(&c.chainStatus, 0) + storePoolChainElt(&d.next, d2) + storePoolChainElt(&c.head, d2) + atomic.StoreInt32(&c.chainStatus, 0) } } } From 5f354158496443e4294c2b104d6dc344236fdcba Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sun, 27 Oct 2019 23:25:12 +0800 Subject: [PATCH 21/34] fix queue bug --- lib/mux/conn.go | 77 +++++++++---------- lib/mux/mux.go | 65 ++++++++-------- lib/mux/mux_test.go | 175 +++++++++++++++++++++++--------------------- lib/mux/queue.go | 152 +++++++++++++++++++++++++++----------- 4 files changed, 275 insertions(+), 194 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 99b6a05..f3217d8 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -4,7 +4,6 @@ import ( "errors" "io" "net" - "strconv" "sync" "sync/atomic" "time" @@ -41,12 +40,12 @@ func NewConn(connId int32, mux *Mux, label ...string) *conn { } c.receiveWindow.New(mux) c.sendWindow.New(mux) - logm := &connLog{ - startTime: time.Now(), - isClose: false, - logs: []string{c.label + "new conn success"}, - } - setM(label[0], int(connId), logm) + //logm := &connLog{ + // startTime: time.Now(), + // isClose: false, + // logs: []string{c.label + "new conn success"}, + //} + //setM(label[0], int(connId), logm) return c } @@ -59,15 +58,15 @@ func (s *conn) Read(buf []byte) (n int, err error) { } // waiting for takeout from receive window finish or timeout n, err = s.receiveWindow.Read(buf, s.connId) - var errstr string - if err == nil { - errstr = "err:nil" - } else { - errstr = err.Error() - } - d := getM(s.label, int(s.connId)) - d.logs = append(d.logs, s.label+"read "+strconv.Itoa(n)+" "+errstr+" "+string(buf[:100])) - setM(s.label, int(s.connId), d) + //var errstr string + //if err == nil { + // errstr = "err:nil" + //} else { + // errstr = err.Error() + //} + //d := getM(s.label, int(s.connId)) + //d.logs = append(d.logs, s.label+"read "+strconv.Itoa(n)+" "+errstr+" "+string(buf[:100])) + //setM(s.label, int(s.connId), d) return } @@ -102,10 +101,10 @@ func (s *conn) closeProcess() { } s.sendWindow.CloseWindow() s.receiveWindow.CloseWindow() - d := getM(s.label, int(s.connId)) - d.isClose = true - d.logs = append(d.logs, s.label+"close "+time.Now().String()) - setM(s.label, int(s.connId), d) + //d := getM(s.label, int(s.connId)) + //d.isClose = true + //d.logs = append(d.logs, s.label+"close "+time.Now().String()) + //setM(s.label, int(s.connId), d) return } @@ -154,12 +153,12 @@ func (Self *window) CloseWindow() { } type ReceiveWindow struct { - bufQueue FIFOQueue + bufQueue ReceiveWindowQueue element *ListElement readLength uint32 readOp chan struct{} readWait bool - windowFull bool + windowFull uint32 count int8 //bw *bandwidth once sync.Once @@ -179,7 +178,7 @@ func (Self *ReceiveWindow) New(mux *Mux) { func (Self *ReceiveWindow) remainingSize() (n uint32) { // receive window remaining - return Self.maxSize - Self.bufQueue.Len() + return atomic.LoadUint32(&Self.maxSize) - Self.bufQueue.Len() } func (Self *ReceiveWindow) readSize() (n uint32) { @@ -207,7 +206,7 @@ func (Self *ReceiveWindow) calcSize() { } // set the maximum size //logs.Warn("n", n) - Self.maxSize = n + atomic.StoreUint32(&Self.maxSize, n) Self.count = -10 } Self.count += 1 @@ -229,7 +228,7 @@ func (Self *ReceiveWindow) Write(buf []byte, l uint16, part bool, id int32) (err Self.calcSize() //logs.Warn("read session calc size finish", Self.maxSize) if Self.remainingSize() == 0 { - Self.windowFull = true + atomic.StoreUint32(&Self.windowFull, 1) //logs.Warn("window full true", Self.windowFull) } Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.readSize()) @@ -259,7 +258,7 @@ copyData: } //logs.Warn("pop element", Self.element.l, Self.element.part) } - l = copy(p[pOff:], Self.element.buf[Self.off:]) + l = copy(p[pOff:], Self.element.buf[Self.off:Self.element.l]) //Self.bw.SetCopySize(l) pOff += l Self.off += uint32(l) @@ -281,13 +280,16 @@ copyData: } func (Self *ReceiveWindow) sendStatus(id int32) { - if Self.windowFull || Self.bufQueue.Len() == 0 { + if Self.bufQueue.Len() == 0 { // window is full before read or empty now - Self.windowFull = false - Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.readSize()) + Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, atomic.LoadUint32(&Self.maxSize), Self.readSize()) // acknowledge other side, have empty some receive window space //} } + if atomic.LoadUint32(&Self.windowFull) > 0 && Self.remainingSize() > 0 { + atomic.StoreUint32(&Self.windowFull, 0) + Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, atomic.LoadUint32(&Self.maxSize), Self.readSize()) + } } func (Self *ReceiveWindow) SetTimeOut(t time.Time) { @@ -309,8 +311,7 @@ type SendWindow struct { buf []byte sentLength uint32 setSizeCh chan struct{} - setSizeWait int32 - unSlide uint32 + setSizeWait uint32 timeout time.Time window } @@ -352,10 +353,10 @@ func (Self *SendWindow) SetSize(windowSize, readLength uint32) (closed bool) { if Self.RemainingSize() == 0 { //logs.Warn("waiting for another window size after slide") // keep the wait status - atomic.StoreInt32(&Self.setSizeWait, 1) + //atomic.StoreUint32(&Self.setSizeWait, 1) return false } - if atomic.CompareAndSwapInt32(&Self.setSizeWait, 1, 0) { + if atomic.CompareAndSwapUint32(&Self.setSizeWait, 1, 0) { // send window into the wait status, need notice the channel select { case Self.setSizeCh <- struct{}{}: @@ -372,7 +373,7 @@ func (Self *SendWindow) SetSize(windowSize, readLength uint32) (closed bool) { func (Self *SendWindow) slide(windowSize, readLength uint32) { atomic.AddUint32(&Self.sentLength, ^readLength-1) - atomic.SwapUint32(&Self.maxSize, windowSize) + atomic.StoreUint32(&Self.maxSize, windowSize) } func (Self *SendWindow) WriteTo() (p []byte, sendSize uint32, part bool, err error) { @@ -386,7 +387,7 @@ func (Self *SendWindow) WriteTo() (p []byte, sendSize uint32, part bool, err err // send window buff is drain, return eof and get another one } if Self.RemainingSize() == 0 { - atomic.StoreInt32(&Self.setSizeWait, 1) + atomic.StoreUint32(&Self.setSizeWait, 1) // into the wait status err = Self.waitReceiveWindow() if err != nil { @@ -395,20 +396,20 @@ func (Self *SendWindow) WriteTo() (p []byte, sendSize uint32, part bool, err err } if len(Self.buf[Self.off:]) > common.MAXIMUM_SEGMENT_SIZE { sendSize = common.MAXIMUM_SEGMENT_SIZE - part = true //logs.Warn("cut buf by mss") } else { sendSize = uint32(len(Self.buf[Self.off:])) - part = false } if Self.RemainingSize() < sendSize { // usable window size is small than // window MAXIMUM_SEGMENT_SIZE or send buf left sendSize = Self.RemainingSize() //logs.Warn("cut buf by remainingsize", sendSize, len(Self.buf[Self.off:])) - part = true } //logs.Warn("send size", sendSize) + if sendSize < uint32(len(Self.buf[Self.off:])) { + part = true + } p = Self.buf[Self.off : sendSize+Self.off] Self.off += sendSize atomic.AddUint32(&Self.sentLength, sendSize) diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 8300977..8a6086c 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -1,7 +1,6 @@ package mux import ( - "bytes" "errors" "io" "math" @@ -29,7 +28,7 @@ type Mux struct { pingTimer *time.Timer connType string writeQueue PriorityQueue - bufCh chan *bytes.Buffer + //bufQueue BytesQueue sync.Mutex } @@ -40,16 +39,16 @@ func NewMux(c net.Conn, connType string) *Mux { conn: c, connMap: NewConnMap(), id: 0, - closeChan: make(chan struct{}, 3), + closeChan: make(chan struct{}, 1), newConnCh: make(chan *conn, 10), bw: new(bandwidth), IsClose: false, connType: connType, - bufCh: make(chan *bytes.Buffer), pingCh: make(chan []byte), pingTimer: time.NewTimer(15 * time.Second), } m.writeQueue.New() + //m.bufQueue.New() //read session by flag m.readSession() //ping @@ -111,16 +110,18 @@ func (s *Mux) sendInfo(flag uint8, id int32, data ...interface{}) { func (s *Mux) writeSession() { go s.packBuf() - go s.writeBuf() + //go s.writeBuf() } func (s *Mux) packBuf() { + buffer := common.BuffPool.Get() for { if s.IsClose { break } + buffer.Reset() pack := s.writeQueue.Pop() - buffer := common.BuffPool.Get() + //buffer := common.BuffPool.Get() err := pack.Pack(buffer) common.MuxPack.Put(pack) if err != nil { @@ -129,34 +130,37 @@ func (s *Mux) packBuf() { break } //logs.Warn(buffer.String()) - select { - case s.bufCh <- buffer: - case <-s.closeChan: + //s.bufQueue.Push(buffer) + l := buffer.Len() + n, err := buffer.WriteTo(s.conn) + //common.BuffPool.Put(buffer) + if err != nil || int(n) != l { + logs.Warn("close from write session fail ", err, n, l) + s.Close() break } } } -func (s *Mux) writeBuf() { - for { - if s.IsClose { - break - } - select { - case buffer := <-s.bufCh: - l := buffer.Len() - n, err := buffer.WriteTo(s.conn) - common.BuffPool.Put(buffer) - if err != nil || int(n) != l { - logs.Warn("close from write session fail ", err, n, l) - s.Close() - break - } - case <-s.closeChan: - break - } - } -} +//func (s *Mux) writeBuf() { +// for { +// if s.IsClose { +// break +// } +// buffer, err := s.bufQueue.Pop() +// if err != nil { +// break +// } +// l := buffer.Len() +// n, err := buffer.WriteTo(s.conn) +// common.BuffPool.Put(buffer) +// if err != nil || int(n) != l { +// logs.Warn("close from write session fail ", err, n, l) +// s.Close() +// break +// } +// } +//} func (s *Mux) ping() { go func() { @@ -310,8 +314,7 @@ func (s *Mux) Close() error { } s.IsClose = true s.connMap.Close() - s.closeChan <- struct{}{} - s.closeChan <- struct{}{} + //s.bufQueue.Stop() s.closeChan <- struct{}{} close(s.newConnCh) return s.conn.Close() diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index 0ac54d5..e3f9dcc 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -4,12 +4,14 @@ import ( "bufio" "fmt" "github.com/cnlh/nps/lib/common" + "io" "log" "net" "net/http" "net/http/httputil" _ "net/http/pprof" "strconv" + "sync" "testing" "time" "unsafe" @@ -48,42 +50,42 @@ func TestNewMux(t *testing.T) { //c2.(*net.TCPConn).SetReadBuffer(0) //c2.(*net.TCPConn).SetReadBuffer(0) go func(c2 net.Conn, c *conn) { - //wg := sync.WaitGroup{} - //wg.Add(1) - //go func() { - // _, err = common.CopyBuffer(c2, c) - // if err != nil { - // c2.Close() - // c.Close() - // logs.Warn("close npc by copy from nps", err, c.connId) - // } - // wg.Done() - //}() - //wg.Add(1) - //go func() { - // _, err = common.CopyBuffer(c, c2) - // if err != nil { - // c2.Close() - // c.Close() - // logs.Warn("close npc by copy from server", err, c.connId) - // } - // wg.Done() - //}() - ////logs.Warn("npc wait") - //wg.Wait() + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + _, err = common.CopyBuffer(c2, c) + if err != nil { + c2.Close() + c.Close() + //logs.Warn("close npc by copy from nps", err, c.connId) + } + wg.Done() + }() + wg.Add(1) + go func() { + _, err = common.CopyBuffer(c, c2) + if err != nil { + c2.Close() + c.Close() + //logs.Warn("close npc by copy from server", err, c.connId) + } + wg.Done() + }() + //logs.Warn("npc wait") + wg.Wait() }(c2, c.(*conn)) } }() go func() { - //m1 := NewMux(conn1, "tcp") + m1 := NewMux(conn1, "tcp") l, err := net.Listen("tcp", "127.0.0.1:7777") if err != nil { logs.Warn(err) } for { //logs.Warn("nps starting accept") - _, err := l.Accept() + conns, err := l.Accept() if err != nil { logs.Warn(err) continue @@ -91,37 +93,38 @@ func TestNewMux(t *testing.T) { //conns.(*net.TCPConn).SetReadBuffer(0) //conns.(*net.TCPConn).SetReadBuffer(0) //logs.Warn("nps accept success starting new conn") - //tmpCpnn, err := m1.NewConn() - //if err != nil { - // logs.Warn("nps new conn err ", err) - // continue - //} - ////logs.Warn("nps new conn success ", tmpCpnn.connId) - //go func(tmpCpnn *conn, conns net.Conn) { - // //go func() { - // // _, err := common.CopyBuffer(tmpCpnn, conns) - // // if err != nil { - // // conns.Close() - // // tmpCpnn.Close() - // // logs.Warn("close nps by copy from user", tmpCpnn.connId, err) - // // } - // //}() - // ////time.Sleep(time.Second) - // //_, err = common.CopyBuffer(conns, tmpCpnn) - // //if err != nil { - // // conns.Close() - // // tmpCpnn.Close() - // // logs.Warn("close nps by copy from npc ", tmpCpnn.connId, err) - // //} - //}(tmpCpnn, conns) + tmpCpnn, err := m1.NewConn() + if err != nil { + logs.Warn("nps new conn err ", err) + continue + } + //logs.Warn("nps new conn success ", tmpCpnn.connId) + go func(tmpCpnn *conn, conns net.Conn) { + go func() { + _, err := common.CopyBuffer(tmpCpnn, conns) + if err != nil { + conns.Close() + tmpCpnn.Close() + //logs.Warn("close nps by copy from user", tmpCpnn.connId, err) + } + }() + //time.Sleep(time.Second) + _, err = common.CopyBuffer(conns, tmpCpnn) + if err != nil { + conns.Close() + tmpCpnn.Close() + //logs.Warn("close nps by copy from npc ", tmpCpnn.connId, err) + } + }(tmpCpnn, conns) } }() - go NewLogServer() + //go NewLogServer() time.Sleep(time.Second * 5) for i := 0; i < 1000; i++ { go test_raw(i) } + //test_request() for { time.Sleep(time.Second * 5) @@ -154,7 +157,7 @@ func client() { func test_request() { conn, _ := net.Dial("tcp", "127.0.0.1:7777") for { - conn.Write([]byte(`GET /videojs5/video.js HTTP/1.1 + conn.Write([]byte(`GET / HTTP/1.1 Host: 127.0.0.1:7777 Connection: keep-alive @@ -177,39 +180,42 @@ Connection: keep-alive } func test_raw(k int) { - for i := 0; i < 1; i++ { + for i := 0; i < 10; i++ { ti := time.Now() - _, _ = net.Dial("tcp", "127.0.0.1:7777") + conn, err := net.Dial("tcp", "127.0.0.1:7777") + if err != nil { + logs.Warn("conn dial err", err) + } tid := time.Now() - // conn.Write([]byte(`GET / HTTP/1.1 - //Host: 127.0.0.1:7777 - // - // - //`)) - // tiw := time.Now() - //buf := make([]byte, 3572) - //n, err := io.ReadFull(conn, buf) - ////n, err := conn.Read(buf) - //if err != nil { - // logs.Warn("close by read response err", err) - // break - //} - ////logs.Warn(n, string(buf[:50]), "\n--------------\n", string(buf[n-50:n])) - ////time.Sleep(time.Second) - //err = conn.Close() - //if err != nil { - // logs.Warn("close conn err ", err) - //} + conn.Write([]byte(`GET / HTTP/1.1 +Host: 127.0.0.1:7777 + + +`)) + tiw := time.Now() + buf := make([]byte, 3572) + n, err := io.ReadFull(conn, buf) + //n, err := conn.Read(buf) + if err != nil { + logs.Warn("close by read response err", err) + break + } + logs.Warn(n, string(buf[:50]), "\n--------------\n", string(buf[n-50:n])) + //time.Sleep(time.Second) + err = conn.Close() + if err != nil { + logs.Warn("close conn err ", err) + } now := time.Now() du := now.Sub(ti).Seconds() dud := now.Sub(tid).Seconds() - //duw := now.Sub(tiw).Seconds() - //if du > 1 { - logs.Warn("duration long", du, dud, k, i) - //} - //if n != 3572 { - // logs.Warn("n loss", n, string(buf)) - //} + duw := now.Sub(tiw).Seconds() + if du > 1 { + logs.Warn("duration long", du, dud, duw, k, i) + } + if n != 3572 { + logs.Warn("n loss", n, string(buf)) + } } } @@ -293,11 +299,11 @@ func TestFIFO(t *testing.T) { logs.EnableFuncCallDepth(true) logs.SetLogFuncCallDepth(3) time.Sleep(time.Second * 5) - d := new(FIFOQueue) + d := new(ReceiveWindowQueue) d.New() go func() { time.Sleep(time.Second) - for i := 0; i < 30000; i++ { + for i := 0; i < 30010; i++ { data, err := d.Pop() if err == nil { //fmt.Println(i, string(data.buf), err) @@ -306,7 +312,9 @@ func TestFIFO(t *testing.T) { //fmt.Println("err", err) logs.Warn("err", err) } + //logs.Warn(d.Len()) } + logs.Warn("pop finish") }() go func() { time.Sleep(time.Second * 10) @@ -314,10 +322,10 @@ func TestFIFO(t *testing.T) { go func(i int) { for n := 0; n < 10; n++ { data := new(ListElement) - by := []byte("test " + strconv.Itoa(i) + strconv.Itoa(n)) + by := []byte("test " + strconv.Itoa(i) + " " + strconv.Itoa(n)) // _ = data.New(by, uint16(len(by)), true) //fmt.Println(string((*data).buf), data) - logs.Warn(string((*data).buf), data) + //logs.Warn(string((*data).buf), data) d.Push(data) } }(i) @@ -337,11 +345,12 @@ func TestPriority(t *testing.T) { d.New() go func() { time.Sleep(time.Second) - for i := 0; i < 36000; i++ { + for i := 0; i < 36005; i++ { data := d.Pop() //fmt.Println(i, string(data.buf), err) logs.Warn(i, string(data.Content), data) } + logs.Warn("pop finish") }() go func() { time.Sleep(time.Second * 10) diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 488c616..6ed2dd6 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -1,10 +1,12 @@ package mux import ( + "bytes" "errors" "github.com/cnlh/nps/lib/common" "io" "math" + "runtime" "sync/atomic" "time" "unsafe" @@ -13,7 +15,7 @@ import ( type QueueOp struct { readOp chan struct{} cleanOp chan struct{} - popWait int32 + popWait uint32 } func (Self *QueueOp) New() { @@ -22,7 +24,7 @@ func (Self *QueueOp) New() { } func (Self *QueueOp) allowPop() (closed bool) { - if atomic.CompareAndSwapInt32(&Self.popWait, 1, 0) { + if atomic.CompareAndSwapUint32(&Self.popWait, 1, 0) { select { case Self.readOp <- struct{}{}: return false @@ -44,7 +46,7 @@ type PriorityQueue struct { highestChain *bufChain middleChain *bufChain lowestChain *bufChain - hunger uint8 + starving uint8 } func (Self *PriorityQueue) New() { @@ -73,7 +75,7 @@ func (Self *PriorityQueue) Push(packager *common.MuxPackager) { return } -const maxHunger uint8 = 10 +const maxStarving uint8 = 8 func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { startPop: @@ -82,31 +84,32 @@ startPop: packager = (*common.MuxPackager)(ptr) return } - if Self.hunger < maxHunger { + if Self.starving < maxStarving { ptr, ok = Self.middleChain.popTail() if ok { packager = (*common.MuxPackager)(ptr) - Self.hunger++ + Self.starving++ return } } ptr, ok = Self.lowestChain.popTail() if ok { packager = (*common.MuxPackager)(ptr) - if Self.hunger > 0 { - Self.hunger = uint8(Self.hunger / 2) + if Self.starving > 0 { + Self.starving = uint8(Self.starving / 2) } return } - if Self.hunger > 0 { + if Self.starving > 0 { ptr, ok = Self.middleChain.popTail() if ok { packager = (*common.MuxPackager)(ptr) + Self.starving++ return } } // PriorityQueue is empty, notice Push method - if atomic.CompareAndSwapInt32(&Self.popWait, 0, 1) { + if atomic.CompareAndSwapUint32(&Self.popWait, 0, 1) { select { case <-Self.readOp: goto startPop @@ -133,7 +136,7 @@ func (Self *ListElement) New(buf []byte, l uint16, part bool) (err error) { return nil } -type FIFOQueue struct { +type ReceiveWindowQueue struct { QueueOp chain *bufChain length uint32 @@ -141,21 +144,21 @@ type FIFOQueue struct { timeout time.Time } -func (Self *FIFOQueue) New() { +func (Self *ReceiveWindowQueue) New() { Self.QueueOp.New() Self.chain = new(bufChain) Self.chain.new(64) Self.stopOp = make(chan struct{}, 1) } -func (Self *FIFOQueue) Push(element *ListElement) { +func (Self *ReceiveWindowQueue) Push(element *ListElement) { Self.chain.pushHead(unsafe.Pointer(element)) atomic.AddUint32(&Self.length, uint32(element.l)) Self.allowPop() return } -func (Self *FIFOQueue) Pop() (element *ListElement, err error) { +func (Self *ReceiveWindowQueue) Pop() (element *ListElement, err error) { startPop: ptr, ok := Self.chain.popTail() if ok { @@ -163,7 +166,7 @@ startPop: atomic.AddUint32(&Self.length, ^uint32(element.l-1)) return } - if atomic.CompareAndSwapInt32(&Self.popWait, 0, 1) { + if atomic.CompareAndSwapUint32(&Self.popWait, 0, 1) { t := Self.timeout.Sub(time.Now()) if t <= 0 { t = time.Minute @@ -186,18 +189,62 @@ startPop: goto startPop } -func (Self *FIFOQueue) Len() (n uint32) { +func (Self *ReceiveWindowQueue) Len() (n uint32) { return atomic.LoadUint32(&Self.length) } -func (Self *FIFOQueue) Stop() { +func (Self *ReceiveWindowQueue) Stop() { Self.stopOp <- struct{}{} } -func (Self *FIFOQueue) SetTimeOut(t time.Time) { +func (Self *ReceiveWindowQueue) SetTimeOut(t time.Time) { Self.timeout = t } +type BytesQueue struct { + QueueOp + chain *bufChain + stopOp chan struct{} +} + +func (Self *BytesQueue) New() { + Self.QueueOp.New() + Self.chain = new(bufChain) + Self.chain.new(8) + Self.stopOp = make(chan struct{}, 1) +} + +func (Self *BytesQueue) Push(buf *bytes.Buffer) { + Self.chain.pushHead(unsafe.Pointer(buf)) + Self.allowPop() + return +} + +func (Self *BytesQueue) Pop() (buf *bytes.Buffer, err error) { +startPop: + ptr, ok := Self.chain.popTail() + if ok { + buf = (*bytes.Buffer)(ptr) + return + } + if atomic.CompareAndSwapUint32(&Self.popWait, 0, 1) { + select { + case <-Self.readOp: + goto startPop + case <-Self.cleanOp: + return + case <-Self.stopOp: + err = io.EOF + return + } + } + goto startPop +} + +func (Self *BytesQueue) Stop() { + Self.stopOp <- struct{}{} +} + // https://golang.org/src/sync/poolqueue.go type bufDequeue struct { @@ -224,7 +271,8 @@ type bufDequeue struct { // index has moved beyond it and typ has been set to nil. This // is set to nil atomically by the consumer and read // atomically by the producer. - vals []unsafe.Pointer + vals []unsafe.Pointer + starving uint32 } const dequeueBits = 32 @@ -253,6 +301,10 @@ func (d *bufDequeue) pack(head, tail uint32) uint64 { // queue is full. func (d *bufDequeue) pushHead(val unsafe.Pointer) bool { var slot *unsafe.Pointer + var starve uint8 + if atomic.LoadUint32(&d.starving) > 0 { + runtime.Gosched() + } for { ptrs := atomic.LoadUint64(&d.headTail) head, tail := d.unpack(ptrs) @@ -263,8 +315,15 @@ func (d *bufDequeue) pushHead(val unsafe.Pointer) bool { ptrs2 := d.pack(head+1, tail) if atomic.CompareAndSwapUint64(&d.headTail, ptrs, ptrs2) { slot = &d.vals[head&uint32(len(d.vals)-1)] + if starve >= 3 && atomic.LoadUint32(&d.starving) > 0 { + atomic.StoreUint32(&d.starving, 0) + } break } + starve++ + if starve >= 3 { + atomic.StoreUint32(&d.starving, 1) + } } // The head slot is free, so we own it. *slot = val @@ -321,8 +380,8 @@ type bufChain struct { // tail is the bufDequeue to popTail from. This is accessed // by consumers, so reads and writes must be atomic. - tail *bufChainElt - chainStatus int32 + tail *bufChainElt + newChain uint32 } type bufChainElt struct { @@ -359,30 +418,39 @@ func (c *bufChain) new(initSize int) { } func (c *bufChain) pushHead(val unsafe.Pointer) { +startPush: for { - d := loadPoolChainElt(&c.head) - - if d.pushHead(val) { - return - } - - // The current dequeue is full. Allocate a new one of twice - // the size. - if atomic.CompareAndSwapInt32(&c.chainStatus, 0, 1) { - newSize := len(d.vals) * 2 - if newSize >= dequeueLimit { - // Can't make it any bigger. - newSize = dequeueLimit - } - - d2 := &bufChainElt{prev: d} - d2.vals = make([]unsafe.Pointer, newSize) - d2.pushHead(val) - storePoolChainElt(&d.next, d2) - storePoolChainElt(&c.head, d2) - atomic.StoreInt32(&c.chainStatus, 0) + if atomic.LoadUint32(&c.newChain) > 0 { + runtime.Gosched() + } else { + break } } + + d := loadPoolChainElt(&c.head) + + if d.pushHead(val) { + return + } + + // The current dequeue is full. Allocate a new one of twice + // the size. + if atomic.CompareAndSwapUint32(&c.newChain, 0, 1) { + newSize := len(d.vals) * 2 + if newSize >= dequeueLimit { + // Can't make it any bigger. + newSize = dequeueLimit + } + + d2 := &bufChainElt{prev: d} + d2.vals = make([]unsafe.Pointer, newSize) + d2.pushHead(val) + storePoolChainElt(&c.head, d2) + storePoolChainElt(&d.next, d2) + atomic.StoreUint32(&c.newChain, 0) + return + } + goto startPush } func (c *bufChain) popTail() (unsafe.Pointer, bool) { From 5f58c34c8bd53c584f832ace882fe7d175508994 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sat, 2 Nov 2019 22:59:52 +0800 Subject: [PATCH 22/34] perf test --- go.mod | 1 + go.sum | 2 + lib/common/pool.go | 53 ++++++++++++++ lib/mux/conn.go | 4 +- lib/mux/mux.go | 11 ++- lib/mux/mux_test.go | 122 ++++++++++++++++--------------- lib/mux/queue.go | 172 ++++++++++++++++++++------------------------ 7 files changed, 207 insertions(+), 158 deletions(-) diff --git a/go.mod b/go.mod index 8a19eaf..a540dae 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/klauspost/cpuid v1.2.1 // indirect github.com/klauspost/reedsolomon v1.9.2 // indirect github.com/onsi/gomega v1.5.0 // indirect + github.com/panjf2000/ants/v2 v2.2.2 github.com/pkg/errors v0.8.0 github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 // indirect github.com/shirou/gopsutil v2.18.12+incompatible diff --git a/go.sum b/go.sum index f3a17f4..f0c4d7f 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,8 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/panjf2000/ants/v2 v2.2.2 h1:TWzusBjq/IflXhy+/S6u5wmMLCBdJnB9tPIx9Zmhvok= +github.com/panjf2000/ants/v2 v2.2.2/go.mod h1:1GFm8bV8nyCQvU5K4WvBCTG1/YBFOD2VzjffD8fV55A= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/lib/common/pool.go b/lib/common/pool.go index 240f7f9..5010012 100644 --- a/lib/common/pool.go +++ b/lib/common/pool.go @@ -2,6 +2,8 @@ package common import ( "bytes" + "github.com/panjf2000/ants/v2" + "net" "sync" ) @@ -149,11 +151,62 @@ func (Self *muxPackagerPool) Put(pack *MuxPackager) { Self.pool.Put(pack) } +type connGroup struct { + src net.Conn + dst net.Conn + wg *sync.WaitGroup +} + +func newConnGroup(src net.Conn, dst net.Conn, wg *sync.WaitGroup) connGroup { + return connGroup{ + src: src, + dst: dst, + wg: wg, + } +} + +func copyConnGroup(group interface{}) { + cg, ok := group.(connGroup) + if !ok { + return + } + _, err := CopyBuffer(cg.src, cg.dst) + if err != nil { + cg.src.Close() + cg.dst.Close() + //logs.Warn("close npc by copy from nps", err, c.connId) + } + cg.wg.Done() +} + +type Conns struct { + conn1 net.Conn + conn2 net.Conn +} + +func NewConns(c1 net.Conn, c2 net.Conn) Conns { + return Conns{ + conn1: c1, + conn2: c2, + } +} + +func copyConns(group interface{}) { + conns := group.(Conns) + wg := new(sync.WaitGroup) + wg.Add(2) + _ = connCopyPool.Invoke(newConnGroup(conns.conn1, conns.conn2, wg)) + _ = connCopyPool.Invoke(newConnGroup(conns.conn2, conns.conn1, wg)) + wg.Wait() +} + var once = sync.Once{} var BuffPool = bufferPool{} var CopyBuff = copyBufferPool{} var MuxPack = muxPackagerPool{} var WindowBuff = windowBufferPool{} +var connCopyPool, _ = ants.NewPoolWithFunc(200000, copyConnGroup, ants.WithNonblocking(false)) +var CopyConnsPool, _ = ants.NewPoolWithFunc(100000, copyConns, ants.WithNonblocking(false)) func newPool() { BuffPool.New() diff --git a/lib/mux/conn.go b/lib/mux/conn.go index f3217d8..0ba6f90 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -156,7 +156,7 @@ type ReceiveWindow struct { bufQueue ReceiveWindowQueue element *ListElement readLength uint32 - readOp chan struct{} + //readOp chan struct{} readWait bool windowFull uint32 count int8 @@ -167,7 +167,7 @@ type ReceiveWindow struct { func (Self *ReceiveWindow) New(mux *Mux) { // initial a window for receive - Self.readOp = make(chan struct{}) + //Self.readOp = make(chan struct{}) Self.bufQueue.New() //Self.bw = new(bandwidth) Self.element = new(ListElement) diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 8a6086c..6a2d6b6 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -5,7 +5,6 @@ import ( "io" "math" "net" - "sync" "sync/atomic" "time" @@ -29,7 +28,7 @@ type Mux struct { connType string writeQueue PriorityQueue //bufQueue BytesQueue - sync.Mutex + //sync.Mutex } func NewMux(c net.Conn, connType string) *Mux { @@ -216,7 +215,7 @@ func (s *Mux) pingReturn() { if latency < 0.5 && latency > 0 { s.latency = latency } - //logs.Warn("latency", s.latency) + logs.Warn("latency", s.latency) common.WindowBuff.Put(data) } }() @@ -242,15 +241,19 @@ func (s *Mux) readSession() { case common.MUX_NEW_CONN: //new connection connection := NewConn(pack.Id, s, "npc ") s.connMap.Set(pack.Id, connection) //it has been set before send ok + //go func(connection *conn) { s.newConnCh <- connection s.sendInfo(common.MUX_NEW_CONN_OK, connection.connId, nil) + //}(connection) continue case common.MUX_PING_FLAG: //ping s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, pack.Content) common.WindowBuff.Put(pack.Content) continue case common.MUX_PING_RETURN: + //go func(content []byte) { s.pingCh <- pack.Content + //}(pack.Content) continue } if connection, ok := s.connMap.Get(pack.Id); ok && !connection.isClose { @@ -275,8 +278,10 @@ func (s *Mux) readSession() { continue case common.MUX_CONN_CLOSE: //close the connection s.connMap.Delete(pack.Id) + //go func(connection *conn) { connection.closeFlag = true connection.receiveWindow.Stop() // close signal to receive window + //}(connection) continue } } else if pack.Flag == common.MUX_CONN_CLOSE { diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index e3f9dcc..d3061ab 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -11,7 +11,6 @@ import ( "net/http/httputil" _ "net/http/pprof" "strconv" - "sync" "testing" "time" "unsafe" @@ -30,6 +29,7 @@ func TestNewMux(t *testing.T) { logs.SetLogFuncCallDepth(3) server() client() + //poolConnCopy, _ := ants.NewPoolWithFunc(200000, common.copyConn, ants.WithNonblocking(false)) time.Sleep(time.Second * 3) go func() { m2 := NewMux(conn2, "tcp") @@ -49,31 +49,34 @@ func TestNewMux(t *testing.T) { } //c2.(*net.TCPConn).SetReadBuffer(0) //c2.(*net.TCPConn).SetReadBuffer(0) - go func(c2 net.Conn, c *conn) { - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - _, err = common.CopyBuffer(c2, c) - if err != nil { - c2.Close() - c.Close() - //logs.Warn("close npc by copy from nps", err, c.connId) - } - wg.Done() - }() - wg.Add(1) - go func() { - _, err = common.CopyBuffer(c, c2) - if err != nil { - c2.Close() - c.Close() - //logs.Warn("close npc by copy from server", err, c.connId) - } - wg.Done() - }() - //logs.Warn("npc wait") - wg.Wait() - }(c2, c.(*conn)) + _ = common.CopyConnsPool.Invoke(common.NewConns(c2, c)) + //go func(c2 net.Conn, c *conn) { + // wg := new(sync.WaitGroup) + // wg.Add(2) + // _ = poolConnCopy.Invoke(common.newConnGroup(c2, c, wg)) + // //go func() { + // // _, err = common.CopyBuffer(c2, c) + // // if err != nil { + // // c2.Close() + // // c.Close() + // // //logs.Warn("close npc by copy from nps", err, c.connId) + // // } + // // wg.Done() + // //}() + // //wg.Add(1) + // _ = poolConnCopy.Invoke(common.newConnGroup(c, c2, wg)) + // //go func() { + // // _, err = common.CopyBuffer(c, c2) + // // if err != nil { + // // c2.Close() + // // c.Close() + // // //logs.Warn("close npc by copy from server", err, c.connId) + // // } + // // wg.Done() + // //}() + // //logs.Warn("npc wait") + // wg.Wait() + //}(c2, c.(*conn)) } }() @@ -99,23 +102,30 @@ func TestNewMux(t *testing.T) { continue } //logs.Warn("nps new conn success ", tmpCpnn.connId) - go func(tmpCpnn *conn, conns net.Conn) { - go func() { - _, err := common.CopyBuffer(tmpCpnn, conns) - if err != nil { - conns.Close() - tmpCpnn.Close() - //logs.Warn("close nps by copy from user", tmpCpnn.connId, err) - } - }() - //time.Sleep(time.Second) - _, err = common.CopyBuffer(conns, tmpCpnn) - if err != nil { - conns.Close() - tmpCpnn.Close() - //logs.Warn("close nps by copy from npc ", tmpCpnn.connId, err) - } - }(tmpCpnn, conns) + _ = common.CopyConnsPool.Invoke(common.NewConns(tmpCpnn, conns)) + //go func(tmpCpnn *conn, conns net.Conn) { + // wg := new(sync.WaitGroup) + // wg.Add(2) + // _ = poolConnCopy.Invoke(common.newConnGroup(tmpCpnn, conns, wg)) + // //go func() { + // // _, err := common.CopyBuffer(tmpCpnn, conns) + // // if err != nil { + // // conns.Close() + // // tmpCpnn.Close() + // // //logs.Warn("close nps by copy from user", tmpCpnn.connId, err) + // // } + // //}() + // //wg.Add(1) + // _ = poolConnCopy.Invoke(common.newConnGroup(conns, tmpCpnn, wg)) + // //time.Sleep(time.Second) + // //_, err = common.CopyBuffer(conns, tmpCpnn) + // //if err != nil { + // // conns.Close() + // // tmpCpnn.Close() + // // //logs.Warn("close nps by copy from npc ", tmpCpnn.connId, err) + // //} + // wg.Wait() + //}(tmpCpnn, conns) } }() @@ -180,7 +190,7 @@ Connection: keep-alive } func test_raw(k int) { - for i := 0; i < 10; i++ { + for i := 0; i < 1; i++ { ti := time.Now() conn, err := net.Dial("tcp", "127.0.0.1:7777") if err != nil { @@ -303,7 +313,7 @@ func TestFIFO(t *testing.T) { d.New() go func() { time.Sleep(time.Second) - for i := 0; i < 30010; i++ { + for i := 0; i < 300100; i++ { data, err := d.Pop() if err == nil { //fmt.Println(i, string(data.buf), err) @@ -318,17 +328,13 @@ func TestFIFO(t *testing.T) { }() go func() { time.Sleep(time.Second * 10) - for i := 0; i < 3000; i++ { - go func(i int) { - for n := 0; n < 10; n++ { - data := new(ListElement) - by := []byte("test " + strconv.Itoa(i) + " " + strconv.Itoa(n)) // - _ = data.New(by, uint16(len(by)), true) - //fmt.Println(string((*data).buf), data) - //logs.Warn(string((*data).buf), data) - d.Push(data) - } - }(i) + for i := 0; i < 300000; i++ { + data := new(ListElement) + by := []byte("test " + strconv.Itoa(i) + " ") // + _ = data.New(by, uint16(len(by)), true) + //fmt.Println(string((*data).buf), data) + //logs.Warn(string((*data).buf), data) + d.Push(data) } }() time.Sleep(time.Second * 100000) @@ -345,7 +351,7 @@ func TestPriority(t *testing.T) { d.New() go func() { time.Sleep(time.Second) - for i := 0; i < 36005; i++ { + for i := 0; i < 360050; i++ { data := d.Pop() //fmt.Println(i, string(data.buf), err) logs.Warn(i, string(data.Content), data) @@ -354,7 +360,7 @@ func TestPriority(t *testing.T) { }() go func() { time.Sleep(time.Second * 10) - for i := 0; i < 3000; i++ { + for i := 0; i < 30000; i++ { go func(i int) { for n := 0; n < 10; n++ { data := new(common.MuxPackager) diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 6ed2dd6..e3c39a1 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -1,52 +1,24 @@ package mux import ( - "bytes" "errors" "github.com/cnlh/nps/lib/common" "io" "math" "runtime" + "sync" "sync/atomic" "time" "unsafe" ) -type QueueOp struct { - readOp chan struct{} - cleanOp chan struct{} - popWait uint32 -} - -func (Self *QueueOp) New() { - Self.readOp = make(chan struct{}) - Self.cleanOp = make(chan struct{}, 2) -} - -func (Self *QueueOp) allowPop() (closed bool) { - if atomic.CompareAndSwapUint32(&Self.popWait, 1, 0) { - select { - case Self.readOp <- struct{}{}: - return false - case <-Self.cleanOp: - return true - } - } - return -} - -func (Self *QueueOp) Clean() { - Self.cleanOp <- struct{}{} - Self.cleanOp <- struct{}{} - close(Self.cleanOp) -} - type PriorityQueue struct { - QueueOp highestChain *bufChain middleChain *bufChain lowestChain *bufChain starving uint8 + stop bool + cond *sync.Cond } func (Self *PriorityQueue) New() { @@ -56,7 +28,8 @@ func (Self *PriorityQueue) New() { Self.middleChain.new(32) Self.lowestChain = new(bufChain) Self.lowestChain.new(256) - Self.QueueOp.New() + locker := new(sync.Mutex) + Self.cond = sync.NewCond(locker) } func (Self *PriorityQueue) Push(packager *common.MuxPackager) { @@ -71,14 +44,44 @@ func (Self *PriorityQueue) Push(packager *common.MuxPackager) { default: Self.lowestChain.pushHead(unsafe.Pointer(packager)) } - Self.allowPop() + //atomic.AddUint32(&Self.count, 1) + Self.cond.Signal() return } const maxStarving uint8 = 8 func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { -startPop: + // PriorityQueue is empty, notice Push method + var iter bool + for { + packager = Self.pop() + if packager != nil { + return + } + if Self.stop { + return + } + if iter { + break + } + iter = true + runtime.Gosched() + } + Self.cond.L.Lock() + defer Self.cond.L.Unlock() + for packager = Self.pop(); packager == nil; { + if Self.stop { + return + } + Self.cond.Wait() + packager = Self.pop() + } + //atomic.AddUint32(&Self.count, ^uint32(0)) + return +} + +func (Self *PriorityQueue) pop() (packager *common.MuxPackager) { ptr, ok := Self.highestChain.popTail() if ok { packager = (*common.MuxPackager)(ptr) @@ -108,16 +111,12 @@ startPop: return } } - // PriorityQueue is empty, notice Push method - if atomic.CompareAndSwapUint32(&Self.popWait, 0, 1) { - select { - case <-Self.readOp: - goto startPop - case <-Self.cleanOp: - return nil - } - } - goto startPop + return +} + +func (Self *PriorityQueue) Stop() { + Self.stop = true + Self.cond.Broadcast() } type ListElement struct { @@ -137,18 +136,19 @@ func (Self *ListElement) New(buf []byte, l uint16, part bool) (err error) { } type ReceiveWindowQueue struct { - QueueOp chain *bufChain length uint32 stopOp chan struct{} + readOp chan struct{} + popWait uint32 timeout time.Time } func (Self *ReceiveWindowQueue) New() { - Self.QueueOp.New() + Self.readOp = make(chan struct{}) Self.chain = new(bufChain) Self.chain.new(64) - Self.stopOp = make(chan struct{}, 1) + Self.stopOp = make(chan struct{}, 2) } func (Self *ReceiveWindowQueue) Push(element *ListElement) { @@ -158,15 +158,30 @@ func (Self *ReceiveWindowQueue) Push(element *ListElement) { return } -func (Self *ReceiveWindowQueue) Pop() (element *ListElement, err error) { -startPop: +func (Self *ReceiveWindowQueue) pop() (element *ListElement) { ptr, ok := Self.chain.popTail() if ok { element = (*ListElement)(ptr) atomic.AddUint32(&Self.length, ^uint32(element.l-1)) return } + return +} + +func (Self *ReceiveWindowQueue) Pop() (element *ListElement, err error) { + var iter bool +startPop: + element = Self.pop() + if element != nil { + return + } + if !iter { + iter = true + runtime.Gosched() + goto startPop + } if atomic.CompareAndSwapUint32(&Self.popWait, 0, 1) { + iter = false t := Self.timeout.Sub(time.Now()) if t <= 0 { t = time.Minute @@ -176,8 +191,6 @@ startPop: select { case <-Self.readOp: goto startPop - case <-Self.cleanOp: - return case <-Self.stopOp: err = io.EOF return @@ -189,62 +202,31 @@ startPop: goto startPop } +func (Self *ReceiveWindowQueue) allowPop() (closed bool) { + if atomic.CompareAndSwapUint32(&Self.popWait, 1, 0) { + select { + case Self.readOp <- struct{}{}: + return false + case <-Self.stopOp: + return true + } + } + return +} + func (Self *ReceiveWindowQueue) Len() (n uint32) { return atomic.LoadUint32(&Self.length) } func (Self *ReceiveWindowQueue) Stop() { Self.stopOp <- struct{}{} + Self.stopOp <- struct{}{} } func (Self *ReceiveWindowQueue) SetTimeOut(t time.Time) { Self.timeout = t } -type BytesQueue struct { - QueueOp - chain *bufChain - stopOp chan struct{} -} - -func (Self *BytesQueue) New() { - Self.QueueOp.New() - Self.chain = new(bufChain) - Self.chain.new(8) - Self.stopOp = make(chan struct{}, 1) -} - -func (Self *BytesQueue) Push(buf *bytes.Buffer) { - Self.chain.pushHead(unsafe.Pointer(buf)) - Self.allowPop() - return -} - -func (Self *BytesQueue) Pop() (buf *bytes.Buffer, err error) { -startPop: - ptr, ok := Self.chain.popTail() - if ok { - buf = (*bytes.Buffer)(ptr) - return - } - if atomic.CompareAndSwapUint32(&Self.popWait, 0, 1) { - select { - case <-Self.readOp: - goto startPop - case <-Self.cleanOp: - return - case <-Self.stopOp: - err = io.EOF - return - } - } - goto startPop -} - -func (Self *BytesQueue) Stop() { - Self.stopOp <- struct{}{} -} - // https://golang.org/src/sync/poolqueue.go type bufDequeue struct { From f362c96e1ee801e41703de2fea93c3239593122e Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sat, 9 Nov 2019 23:02:29 +0800 Subject: [PATCH 23/34] fine mux, add goroutine pool --- lib/common/pool.go | 67 ++++-------- lib/conn/conn.go | 37 ++++--- lib/goroutine/pool.go | 73 +++++++++++++ lib/mux/conn.go | 247 +++++++++++++++++++++++++----------------- lib/mux/mux.go | 2 +- lib/mux/mux_test.go | 31 +++--- lib/mux/queue.go | 123 +++++++++++---------- 7 files changed, 352 insertions(+), 228 deletions(-) create mode 100644 lib/goroutine/pool.go diff --git a/lib/common/pool.go b/lib/common/pool.go index 5010012..5d4da00 100644 --- a/lib/common/pool.go +++ b/lib/common/pool.go @@ -2,8 +2,6 @@ package common import ( "bytes" - "github.com/panjf2000/ants/v2" - "net" "sync" ) @@ -151,53 +149,34 @@ func (Self *muxPackagerPool) Put(pack *MuxPackager) { Self.pool.Put(pack) } -type connGroup struct { - src net.Conn - dst net.Conn - wg *sync.WaitGroup +type ListElement struct { + Buf []byte + L uint16 + Part bool } -func newConnGroup(src net.Conn, dst net.Conn, wg *sync.WaitGroup) connGroup { - return connGroup{ - src: src, - dst: dst, - wg: wg, +type listElementPool struct { + pool sync.Pool +} + +func (Self *listElementPool) New() { + Self.pool = sync.Pool{ + New: func() interface{} { + element := ListElement{} + return &element + }, } } -func copyConnGroup(group interface{}) { - cg, ok := group.(connGroup) - if !ok { - return - } - _, err := CopyBuffer(cg.src, cg.dst) - if err != nil { - cg.src.Close() - cg.dst.Close() - //logs.Warn("close npc by copy from nps", err, c.connId) - } - cg.wg.Done() +func (Self *listElementPool) Get() *ListElement { + return Self.pool.Get().(*ListElement) } -type Conns struct { - conn1 net.Conn - conn2 net.Conn -} - -func NewConns(c1 net.Conn, c2 net.Conn) Conns { - return Conns{ - conn1: c1, - conn2: c2, - } -} - -func copyConns(group interface{}) { - conns := group.(Conns) - wg := new(sync.WaitGroup) - wg.Add(2) - _ = connCopyPool.Invoke(newConnGroup(conns.conn1, conns.conn2, wg)) - _ = connCopyPool.Invoke(newConnGroup(conns.conn2, conns.conn1, wg)) - wg.Wait() +func (Self *listElementPool) Put(element *ListElement) { + element.L = 0 + element.Buf = nil + element.Part = false + Self.pool.Put(element) } var once = sync.Once{} @@ -205,14 +184,14 @@ var BuffPool = bufferPool{} var CopyBuff = copyBufferPool{} var MuxPack = muxPackagerPool{} var WindowBuff = windowBufferPool{} -var connCopyPool, _ = ants.NewPoolWithFunc(200000, copyConnGroup, ants.WithNonblocking(false)) -var CopyConnsPool, _ = ants.NewPoolWithFunc(100000, copyConns, ants.WithNonblocking(false)) +var ListElementPool = listElementPool{} func newPool() { BuffPool.New() CopyBuff.New() MuxPack.New() WindowBuff.New() + ListElementPool.New() } func init() { diff --git a/lib/conn/conn.go b/lib/conn/conn.go index 7946c0d..9f0c397 100755 --- a/lib/conn/conn.go +++ b/lib/conn/conn.go @@ -6,13 +6,14 @@ import ( "encoding/binary" "encoding/json" "errors" + "github.com/astaxie/beego/logs" + "github.com/cnlh/nps/lib/goroutine" "io" "net" "net/http" "net/url" "strconv" "strings" - "sync" "time" "github.com/cnlh/nps/lib/common" @@ -350,25 +351,29 @@ func SetUdpSession(sess *kcp.UDPSession) { //conn1 mux conn func CopyWaitGroup(conn1, conn2 net.Conn, crypt bool, snappy bool, rate *rate.Rate, flow *file.Flow, isServer bool, rb []byte) { - var in, out int64 - var wg sync.WaitGroup + //var in, out int64 + //var wg sync.WaitGroup connHandle := GetConn(conn1, crypt, snappy, rate, isServer) if rb != nil { connHandle.Write(rb) } - go func(in *int64) { - wg.Add(1) - *in, _ = common.CopyBuffer(connHandle, conn2) - connHandle.Close() - conn2.Close() - wg.Done() - }(&in) - out, _ = common.CopyBuffer(conn2, connHandle) - connHandle.Close() - conn2.Close() - wg.Wait() - if flow != nil { - flow.Add(in, out) + //go func(in *int64) { + // wg.Add(1) + // *in, _ = common.CopyBuffer(connHandle, conn2) + // connHandle.Close() + // conn2.Close() + // wg.Done() + //}(&in) + //out, _ = common.CopyBuffer(conn2, connHandle) + //connHandle.Close() + //conn2.Close() + //wg.Wait() + //if flow != nil { + // flow.Add(in, out) + //} + err := goroutine.CopyConnsPool.Invoke(goroutine.NewConns(connHandle, conn2, flow)) + if err != nil { + logs.Error(err) } } diff --git a/lib/goroutine/pool.go b/lib/goroutine/pool.go new file mode 100644 index 0000000..287c711 --- /dev/null +++ b/lib/goroutine/pool.go @@ -0,0 +1,73 @@ +package goroutine + +import ( + "github.com/cnlh/nps/lib/common" + "github.com/cnlh/nps/lib/file" + "github.com/panjf2000/ants/v2" + "io" + "net" + "sync" +) + +type connGroup struct { + src io.ReadWriteCloser + dst io.ReadWriteCloser + wg *sync.WaitGroup + n *int64 +} + +func newConnGroup(dst, src io.ReadWriteCloser, wg *sync.WaitGroup, n *int64) connGroup { + return connGroup{ + src: src, + dst: dst, + wg: wg, + n: n, + } +} + +func copyConnGroup(group interface{}) { + cg, ok := group.(connGroup) + if !ok { + return + } + var err error + *cg.n, err = common.CopyBuffer(cg.dst, cg.src) + if err != nil { + cg.src.Close() + cg.dst.Close() + //logs.Warn("close npc by copy from nps", err, c.connId) + } + cg.wg.Done() +} + +type Conns struct { + conn1 io.ReadWriteCloser // mux connection + conn2 net.Conn // outside connection + flow *file.Flow +} + +func NewConns(c1 io.ReadWriteCloser, c2 net.Conn, flow *file.Flow) Conns { + return Conns{ + conn1: c1, + conn2: c2, + flow: flow, + } +} + +func copyConns(group interface{}) { + conns := group.(Conns) + wg := new(sync.WaitGroup) + wg.Add(2) + var in, out int64 + _ = connCopyPool.Invoke(newConnGroup(conns.conn1, conns.conn2, wg, &in)) + // outside to mux : incoming + _ = connCopyPool.Invoke(newConnGroup(conns.conn2, conns.conn1, wg, &out)) + // mux to outside : outgoing + wg.Wait() + if conns.flow != nil { + conns.flow.Add(in, out) + } +} + +var connCopyPool, _ = ants.NewPoolWithFunc(200000, copyConnGroup, ants.WithNonblocking(false)) +var CopyConnsPool, _ = ants.NewPoolWithFunc(100000, copyConns, ants.WithNonblocking(false)) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 0ba6f90..cb982e8 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -4,6 +4,7 @@ import ( "errors" "io" "net" + "runtime" "sync" "sync/atomic" "time" @@ -57,7 +58,12 @@ func (s *conn) Read(buf []byte) (n int, err error) { return 0, nil } // waiting for takeout from receive window finish or timeout + //now := time.Now() n, err = s.receiveWindow.Read(buf, s.connId) + //t := time.Now().Sub(now) + //if t.Seconds() > 0.5 { + //logs.Warn("conn read long", n, t.Seconds()) + //} //var errstr string //if err == nil { // errstr = "err:nil" @@ -82,7 +88,12 @@ func (s *conn) Write(buf []byte) (n int, err error) { return 0, nil } //logs.Warn("write buf", len(buf)) + //now := time.Now() n, err = s.sendWindow.WriteFull(buf, s.connId) + //t := time.Now().Sub(now) + //if t.Seconds() > 0.5 { + // logs.Warn("conn write long", n, t.Seconds()) + //} return } @@ -133,11 +144,25 @@ func (s *conn) SetWriteDeadline(t time.Time) error { } type window struct { - off uint32 - maxSize uint32 - closeOp bool - closeOpCh chan struct{} - mux *Mux + off uint32 + maxSize uint32 + closeOp bool + closeOpCh chan struct{} + remainingWait uint64 + mux *Mux +} + +func (Self *window) unpack(ptrs uint64) (remaining, wait uint32) { + const mask = 1<> dequeueBits) & mask) + wait = uint32(ptrs & mask) + return +} + +func (Self *window) pack(remaining, wait uint32) uint64 { + const mask = 1< 0 { + n = uint32(l) + } + return } func (Self *ReceiveWindow) calcSize() { @@ -194,8 +212,9 @@ func (Self *ReceiveWindow) calcSize() { if n < 8192 { n = 8192 } - if n < Self.bufQueue.Len() { - n = Self.bufQueue.Len() + bufLen := Self.bufQueue.Len() + if n < bufLen { + n = bufLen } // set the minimal size if n > 2*Self.maxSize { @@ -210,28 +229,39 @@ func (Self *ReceiveWindow) calcSize() { Self.count = -10 } Self.count += 1 + return } func (Self *ReceiveWindow) Write(buf []byte, l uint16, part bool, id int32) (err error) { if Self.closeOp { return errors.New("conn.receiveWindow: write on closed window") } - element := new(ListElement) - err = element.New(buf, l, part) + element, err := NewListElement(buf, l, part) //logs.Warn("push the buf", len(buf), l, (&element).l) if err != nil { return } - Self.bufQueue.Push(element) // must push data before allow read - //logs.Warn("read session calc size ", Self.maxSize) - // calculating the receive window size - Self.calcSize() - //logs.Warn("read session calc size finish", Self.maxSize) - if Self.remainingSize() == 0 { - atomic.StoreUint32(&Self.windowFull, 1) - //logs.Warn("window full true", Self.windowFull) + Self.calcSize() // calculate the max window size + var wait uint32 +start: + ptrs := atomic.LoadUint64(&Self.remainingWait) + _, wait = Self.unpack(ptrs) + newRemaining := Self.remainingSize(l) + // calculate the remaining window size now, plus the element we will push + if newRemaining == 0 { + //logs.Warn("window full true", remaining) + wait = 1 + } + if !atomic.CompareAndSwapUint64(&Self.remainingWait, ptrs, Self.pack(0, wait)) { + goto start + // another goroutine change the status, make sure shall we need wait + } + Self.bufQueue.Push(element) + // status check finish, now we can push the element into the queue + if wait == 0 { + Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, newRemaining) + // send the remaining window size, not including zero size } - Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.readSize()) return nil } @@ -243,10 +273,10 @@ func (Self *ReceiveWindow) Read(p []byte, id int32) (n int, err error) { l := 0 //logs.Warn("receive window read off, element.l", Self.off, Self.element.l) copyData: - //Self.bw.StartRead() - if Self.off == uint32(Self.element.l) { + if Self.off == uint32(Self.element.L) { // on the first Read method invoked, Self.off and Self.element.l // both zero value + common.ListElementPool.Put(Self.element) Self.element, err = Self.bufQueue.Pop() // if the queue is empty, Pop method will wait until one element push // into the queue successful, or timeout. @@ -258,38 +288,44 @@ copyData: } //logs.Warn("pop element", Self.element.l, Self.element.part) } - l = copy(p[pOff:], Self.element.buf[Self.off:Self.element.l]) - //Self.bw.SetCopySize(l) + l = copy(p[pOff:], Self.element.Buf[Self.off:Self.element.L]) pOff += l Self.off += uint32(l) - atomic.AddUint32(&Self.readLength, uint32(l)) //logs.Warn("window read length buf len", Self.readLength, Self.bufQueue.Len()) n += l l = 0 - //Self.bw.EndRead() - if Self.off == uint32(Self.element.l) { + if Self.off == uint32(Self.element.L) { //logs.Warn("put the element end ", string(Self.element.buf[:15])) - common.WindowBuff.Put(Self.element.buf) - Self.sendStatus(id) + common.WindowBuff.Put(Self.element.Buf) + Self.sendStatus(id, Self.element.L) + // check the window full status } - if pOff < len(p) && Self.element.part { + if pOff < len(p) && Self.element.Part { // element is a part of the segments, trying to fill up buf p goto copyData } return // buf p is full or all of segments in buf, return } -func (Self *ReceiveWindow) sendStatus(id int32) { - if Self.bufQueue.Len() == 0 { - // window is full before read or empty now - Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, atomic.LoadUint32(&Self.maxSize), Self.readSize()) - // acknowledge other side, have empty some receive window space - //} +func (Self *ReceiveWindow) sendStatus(id int32, l uint16) { + var remaining, wait uint32 + for { + ptrs := atomic.LoadUint64(&Self.remainingWait) + remaining, wait = Self.unpack(ptrs) + remaining += uint32(l) + if atomic.CompareAndSwapUint64(&Self.remainingWait, ptrs, Self.pack(remaining, 0)) { + break + } + runtime.Gosched() + // another goroutine change remaining or wait status, make sure + // we need acknowledge other side } - if atomic.LoadUint32(&Self.windowFull) > 0 && Self.remainingSize() > 0 { - atomic.StoreUint32(&Self.windowFull, 0) - Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, atomic.LoadUint32(&Self.maxSize), Self.readSize()) + // now we get the current window status success + if wait == 1 { + //logs.Warn("send the wait status", remaining) + Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, atomic.LoadUint32(&Self.maxSize), remaining) } + return } func (Self *ReceiveWindow) SetTimeOut(t time.Time) { @@ -308,17 +344,16 @@ func (Self *ReceiveWindow) CloseWindow() { } type SendWindow struct { - buf []byte - sentLength uint32 - setSizeCh chan struct{} - setSizeWait uint32 - timeout time.Time + buf []byte + setSizeCh chan struct{} + timeout time.Time window } func (Self *SendWindow) New(mux *Mux) { Self.setSizeCh = make(chan struct{}) Self.maxSize = 4096 + atomic.AddUint64(&Self.remainingWait, uint64(4096)< common.MAXIMUM_SEGMENT_SIZE { sendSize = common.MAXIMUM_SEGMENT_SIZE //logs.Warn("cut buf by mss") } else { sendSize = uint32(len(Self.buf[Self.off:])) } - if Self.RemainingSize() < sendSize { + if remaining < sendSize { // usable window size is small than // window MAXIMUM_SEGMENT_SIZE or send buf left - sendSize = Self.RemainingSize() + sendSize = remaining //logs.Warn("cut buf by remainingsize", sendSize, len(Self.buf[Self.off:])) } //logs.Warn("send size", sendSize) @@ -412,7 +464,7 @@ func (Self *SendWindow) WriteTo() (p []byte, sendSize uint32, part bool, err err } p = Self.buf[Self.off : sendSize+Self.off] Self.off += sendSize - atomic.AddUint32(&Self.sentLength, sendSize) + Self.sent(sendSize) return } @@ -439,6 +491,7 @@ func (Self *SendWindow) waitReceiveWindow() (err error) { func (Self *SendWindow) WriteFull(buf []byte, id int32) (n int, err error) { Self.SetSendBuf(buf) // set the buf to send window + //logs.Warn("set the buf to send window") var bufSeg []byte var part bool var l uint32 diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 6a2d6b6..585c980 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -215,7 +215,7 @@ func (s *Mux) pingReturn() { if latency < 0.5 && latency > 0 { s.latency = latency } - logs.Warn("latency", s.latency) + //logs.Warn("latency", s.latency) common.WindowBuff.Put(data) } }() diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index d3061ab..151def1 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "github.com/cnlh/nps/lib/common" + "github.com/cnlh/nps/lib/goroutine" "io" "log" "net" @@ -49,7 +50,7 @@ func TestNewMux(t *testing.T) { } //c2.(*net.TCPConn).SetReadBuffer(0) //c2.(*net.TCPConn).SetReadBuffer(0) - _ = common.CopyConnsPool.Invoke(common.NewConns(c2, c)) + _ = goroutine.CopyConnsPool.Invoke(goroutine.NewConns(c, c2, nil)) //go func(c2 net.Conn, c *conn) { // wg := new(sync.WaitGroup) // wg.Add(2) @@ -102,7 +103,7 @@ func TestNewMux(t *testing.T) { continue } //logs.Warn("nps new conn success ", tmpCpnn.connId) - _ = common.CopyConnsPool.Invoke(common.NewConns(tmpCpnn, conns)) + _ = goroutine.CopyConnsPool.Invoke(goroutine.NewConns(tmpCpnn, conns, nil)) //go func(tmpCpnn *conn, conns net.Conn) { // wg := new(sync.WaitGroup) // wg.Add(2) @@ -131,9 +132,9 @@ func TestNewMux(t *testing.T) { //go NewLogServer() time.Sleep(time.Second * 5) - for i := 0; i < 1000; i++ { - go test_raw(i) - } + //for i := 0; i < 1; i++ { + // go test_raw(i) + //} //test_request() for { @@ -166,7 +167,7 @@ func client() { func test_request() { conn, _ := net.Dial("tcp", "127.0.0.1:7777") - for { + for i := 0; i < 1000; i++ { conn.Write([]byte(`GET / HTTP/1.1 Host: 127.0.0.1:7777 Connection: keep-alive @@ -185,19 +186,20 @@ Connection: keep-alive break } fmt.Println(string(b[:20]), err) - time.Sleep(time.Second) + //time.Sleep(time.Second) } + logs.Warn("finish") } func test_raw(k int) { - for i := 0; i < 1; i++ { + for i := 0; i < 1000; i++ { ti := time.Now() conn, err := net.Dial("tcp", "127.0.0.1:7777") if err != nil { logs.Warn("conn dial err", err) } tid := time.Now() - conn.Write([]byte(`GET / HTTP/1.1 + conn.Write([]byte(`GET /videojs5/video.js HTTP/1.1 Host: 127.0.0.1:7777 @@ -227,6 +229,7 @@ Host: 127.0.0.1:7777 logs.Warn("n loss", n, string(buf)) } } + logs.Warn("finish") } func TestNewConn(t *testing.T) { @@ -313,11 +316,12 @@ func TestFIFO(t *testing.T) { d.New() go func() { time.Sleep(time.Second) - for i := 0; i < 300100; i++ { + for i := 0; i < 1001; i++ { data, err := d.Pop() if err == nil { //fmt.Println(i, string(data.buf), err) - logs.Warn(i, string(data.buf), err) + logs.Warn(i, string(data.Buf), err) + common.ListElementPool.Put(data) } else { //fmt.Println("err", err) logs.Warn("err", err) @@ -328,10 +332,9 @@ func TestFIFO(t *testing.T) { }() go func() { time.Sleep(time.Second * 10) - for i := 0; i < 300000; i++ { - data := new(ListElement) + for i := 0; i < 1000; i++ { by := []byte("test " + strconv.Itoa(i) + " ") // - _ = data.New(by, uint16(len(by)), true) + data, _ := NewListElement(by, uint16(len(by)), true) //fmt.Println(string((*data).buf), data) //logs.Warn(string((*data).buf), data) d.Push(data) diff --git a/lib/mux/queue.go b/lib/mux/queue.go index e3c39a1..4790779 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -44,7 +44,6 @@ func (Self *PriorityQueue) Push(packager *common.MuxPackager) { default: Self.lowestChain.pushHead(unsafe.Pointer(packager)) } - //atomic.AddUint32(&Self.count, 1) Self.cond.Signal() return } @@ -52,7 +51,6 @@ func (Self *PriorityQueue) Push(packager *common.MuxPackager) { const maxStarving uint8 = 8 func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { - // PriorityQueue is empty, notice Push method var iter bool for { packager = Self.pop() @@ -64,6 +62,7 @@ func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { } if iter { break + // trying to pop twice } iter = true runtime.Gosched() @@ -74,10 +73,12 @@ func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { if Self.stop { return } + //logs.Warn("queue into wait") Self.cond.Wait() + // wait for it with no more iter packager = Self.pop() + //logs.Warn("queue wait finish", packager) } - //atomic.AddUint32(&Self.count, ^uint32(0)) return } @@ -88,6 +89,7 @@ func (Self *PriorityQueue) pop() (packager *common.MuxPackager) { return } if Self.starving < maxStarving { + // not pop too much, lowestChain will wait too long ptr, ok = Self.middleChain.popTail() if ok { packager = (*common.MuxPackager)(ptr) @@ -119,29 +121,27 @@ func (Self *PriorityQueue) Stop() { Self.cond.Broadcast() } -type ListElement struct { - buf []byte - l uint16 - part bool -} - -func (Self *ListElement) New(buf []byte, l uint16, part bool) (err error) { +func NewListElement(buf []byte, l uint16, part bool) (element *common.ListElement, err error) { if uint16(len(buf)) != l { - return errors.New("ListElement: buf length not match") + err = errors.New("ListElement: buf length not match") + return } - Self.buf = buf - Self.l = l - Self.part = part - return nil + //if l == 0 { + // logs.Warn("push zero") + //} + element = common.ListElementPool.Get() + element.Buf = buf + element.L = l + element.Part = part + return } type ReceiveWindowQueue struct { - chain *bufChain - length uint32 - stopOp chan struct{} - readOp chan struct{} - popWait uint32 - timeout time.Time + chain *bufChain + stopOp chan struct{} + readOp chan struct{} + lengthWait uint64 + timeout time.Time } func (Self *ReceiveWindowQueue) New() { @@ -151,45 +151,45 @@ func (Self *ReceiveWindowQueue) New() { Self.stopOp = make(chan struct{}, 2) } -func (Self *ReceiveWindowQueue) Push(element *ListElement) { +func (Self *ReceiveWindowQueue) Push(element *common.ListElement) { + var length, wait uint32 + for { + ptrs := atomic.LoadUint64(&Self.lengthWait) + length, wait = Self.chain.head.unpack(ptrs) + length += uint32(element.L) + if atomic.CompareAndSwapUint64(&Self.lengthWait, ptrs, Self.chain.head.pack(length, 0)) { + break + } + // another goroutine change the length or into wait, make sure + } + //logs.Warn("window push before", Self.Len(), uint32(element.l), len(element.buf)) Self.chain.pushHead(unsafe.Pointer(element)) - atomic.AddUint32(&Self.length, uint32(element.l)) - Self.allowPop() - return -} - -func (Self *ReceiveWindowQueue) pop() (element *ListElement) { - ptr, ok := Self.chain.popTail() - if ok { - element = (*ListElement)(ptr) - atomic.AddUint32(&Self.length, ^uint32(element.l-1)) - return + //logs.Warn("window push", Self.Len()) + if wait == 1 { + Self.allowPop() } return } -func (Self *ReceiveWindowQueue) Pop() (element *ListElement, err error) { - var iter bool +func (Self *ReceiveWindowQueue) Pop() (element *common.ListElement, err error) { + var length uint32 startPop: - element = Self.pop() - if element != nil { - return - } - if !iter { - iter = true - runtime.Gosched() - goto startPop - } - if atomic.CompareAndSwapUint32(&Self.popWait, 0, 1) { - iter = false + ptrs := atomic.LoadUint64(&Self.lengthWait) + length, _ = Self.chain.head.unpack(ptrs) + if length == 0 { + if !atomic.CompareAndSwapUint64(&Self.lengthWait, ptrs, Self.chain.head.pack(0, 1)) { + goto startPop // another goroutine is pushing + } t := Self.timeout.Sub(time.Now()) if t <= 0 { t = time.Minute } timer := time.NewTimer(t) defer timer.Stop() + //logs.Warn("queue into wait") select { case <-Self.readOp: + //logs.Warn("queue wait finish") goto startPop case <-Self.stopOp: err = io.EOF @@ -199,23 +199,34 @@ startPop: return } } - goto startPop + // length is not zero, so try to pop + for { + ptr, ok := Self.chain.popTail() + if ok { + //logs.Warn("window pop before", Self.Len()) + element = (*common.ListElement)(ptr) + atomic.AddUint64(&Self.lengthWait, ^(uint64(element.L)< Date: Sat, 9 Nov 2019 23:24:26 +0800 Subject: [PATCH 24/34] version --- lib/version/version.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/version/version.go b/lib/version/version.go index 4cc0532..1f14ef1 100644 --- a/lib/version/version.go +++ b/lib/version/version.go @@ -1,8 +1,8 @@ package version -const VERSION = "0.23.3" +const VERSION = "0.24.0" // Compulsory minimum version, Minimum downward compatibility to this version func GetVersion() string { - return "0.22.0" + return "0.24.0" } From bc1783cfb64c7e882eef7ac84580bb3bc7044971 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sun, 10 Nov 2019 21:04:35 +0800 Subject: [PATCH 25/34] 64bit alignment, readme --- README.md | 28 ++++++++++++++++++++++++++++ lib/mux/conn.go | 6 +++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f56c32a..4a9ddd4 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,7 @@ nps是一款轻量级、高性能、功能强大的**内网穿透**代理服务 * [获取用户真实ip](#获取用户真实ip) * [客户端地址显示](#客户端地址显示) * [客户端与服务端版本对比](#客户端与服务端版本对比) + * [Linux系统限制](#Linux系统限制) * [webAPI](#webAPI) * [贡献](#贡献) * [支持nps发展](#捐赠) @@ -144,6 +145,7 @@ nps是一款轻量级、高性能、功能强大的**内网穿透**代理服务 ```shell ./npc -server=1.1.1.1:8284 -vkey=客户端的密钥 ``` +**注意：运行服务端后，请确保能从客户端设备上正常访问配置文件中所配置的`bridge_port`端口，telnet，netcat这类的来检查** ### 域名解析 @@ -441,6 +443,27 @@ server_ip=xxx ``` ./npc -config=npc配置文件路径 ``` +可自行添加systemd service，例如：`npc.service` +``` +[Unit] +Description=npc - convenient proxy server client +Documentation=https://github.com/cnlh/nps/ +After=network-online.target remote-fs.target nss-lookup.target +Wants=network-online.target + +[Service] +Type=simple +KillMode=process +Restart=always +RestartSec=15s +StandardOutput=append:/var/log/nps/npc.log +ExecStartPre=/bin/echo 'Starting npc' +ExecStopPost=/bin/echo 'Stopping npc' +ExecStart=/absolutely path to/npc -server=ip:port -vkey=web界面中显示的密钥 + +[Install] +WantedBy=multi-user.target +``` #### 配置文件说明 [示例配置文件](https://github.com/cnlh/nps/tree/master/conf/npc.conf) ##### 全局配置 @@ -909,6 +932,11 @@ LevelInformational->6 LevelDebug->7 ### 客户端与服务端版本对比 为了程序正常运行，客户端与服务端的核心版本必须一致，否则将导致客户端无法成功连接致服务端。 +### Linux系统限制 +默认情况下linux对连接数量有限制，对于性能好的机器完全可以调整内核参数以处理更多的连接。 +`tcp_max_syn_backlog` `somaxconn` +酌情调整参数，增强网络性能 + ## webAPI ### webAPI验证说明 diff --git a/lib/mux/conn.go b/lib/mux/conn.go index cb982e8..7d2e351 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -144,11 +144,11 @@ func (s *conn) SetWriteDeadline(t time.Time) error { } type window struct { + remainingWait uint64 // 64bit alignment off uint32 maxSize uint32 closeOp bool closeOpCh chan struct{} - remainingWait uint64 mux *Mux } @@ -178,11 +178,11 @@ func (Self *window) CloseWindow() { } type ReceiveWindow struct { + window bufQueue ReceiveWindowQueue element *common.ListElement count int8 once sync.Once - window } func (Self *ReceiveWindow) New(mux *Mux) { @@ -344,10 +344,10 @@ func (Self *ReceiveWindow) CloseWindow() { } type SendWindow struct { + window buf []byte setSizeCh chan struct{} timeout time.Time - window } func (Self *SendWindow) New(mux *Mux) { From 2ca84c912b32e73ddae30916aa191caa973c3a52 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Wed, 13 Nov 2019 23:33:02 +0800 Subject: [PATCH 26/34] fix mux auto disconnect, add mux new connection queue --- lib/mux/conn.go | 8 ++-- lib/mux/mux.go | 81 +++++++++++++++++--------------- lib/mux/queue.go | 119 ++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 149 insertions(+), 59 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 7d2e351..23bc5d5 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -23,7 +23,7 @@ type conn struct { receiveWindow *ReceiveWindow sendWindow *SendWindow once sync.Once - label string + //label string } func NewConn(connId int32, mux *Mux, label ...string) *conn { @@ -36,9 +36,9 @@ func NewConn(connId int32, mux *Mux, label ...string) *conn { sendWindow: new(SendWindow), once: sync.Once{}, } - if len(label) > 0 { - c.label = label[0] - } + //if len(label) > 0 { + // c.label = label[0] + //} c.receiveWindow.New(mux) c.sendWindow.New(mux) //logm := &connLog{ diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 585c980..b64243f 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -14,21 +14,20 @@ import ( type Mux struct { net.Listener - conn net.Conn - connMap *connMap - newConnCh chan *conn - id int32 - closeChan chan struct{} - IsClose bool - pingOk int - latency float64 - bw *bandwidth - pingCh chan []byte - pingTimer *time.Timer - connType string - writeQueue PriorityQueue - //bufQueue BytesQueue - //sync.Mutex + conn net.Conn + connMap *connMap + newConnCh chan *conn + id int32 + closeChan chan struct{} + IsClose bool + pingOk int + latency float64 + bw *bandwidth + pingCh chan []byte + pingCheck bool + connType string + writeQueue PriorityQueue + newConnQueue ConnQueue } func NewMux(c net.Conn, connType string) *Mux { @@ -39,15 +38,14 @@ func NewMux(c net.Conn, connType string) *Mux { connMap: NewConnMap(), id: 0, closeChan: make(chan struct{}, 1), - newConnCh: make(chan *conn, 10), + newConnCh: make(chan *conn), bw: new(bandwidth), IsClose: false, connType: connType, pingCh: make(chan []byte), - pingTimer: time.NewTimer(15 * time.Second), } m.writeQueue.New() - //m.bufQueue.New() + m.newConnQueue.New() //read session by flag m.readSession() //ping @@ -101,6 +99,8 @@ func (s *Mux) sendInfo(flag uint8, id int32, data ...interface{}) { err = pack.NewPac(flag, id, data...) if err != nil { common.MuxPack.Put(pack) + logs.Error("mux: new pack err") + s.Close() return } s.writeQueue.Push(pack) @@ -124,7 +124,7 @@ func (s *Mux) packBuf() { err := pack.Pack(buffer) common.MuxPack.Put(pack) if err != nil { - logs.Warn("pack err", err) + logs.Error("mux: pack err", err) common.BuffPool.Put(buffer) break } @@ -134,7 +134,7 @@ func (s *Mux) packBuf() { n, err := buffer.WriteTo(s.conn) //common.BuffPool.Put(buffer) if err != nil || int(n) != l { - logs.Warn("close from write session fail ", err, n, l) + logs.Error("mux: close from write session fail ", err, n, l) s.Close() break } @@ -170,21 +170,23 @@ func (s *Mux) ping() { for { if s.IsClose { ticker.Stop() - if !s.pingTimer.Stop() { - <-s.pingTimer.C - } break } select { case <-ticker.C: } + if s.pingCheck { + logs.Error("mux: ping time out") + s.Close() + // more than 5 seconds not receive the ping return package, + // mux conn is damaged, maybe a packet drop, close it + break + } now, _ := time.Now().UTC().MarshalText() s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) - if !s.pingTimer.Stop() { - <-s.pingTimer.C - } - s.pingTimer.Reset(15 * time.Second) + s.pingCheck = true if s.pingOk > 10 && s.connType == "kcp" { + logs.Error("mux: kcp ping err") s.Close() break } @@ -203,12 +205,9 @@ func (s *Mux) pingReturn() { } select { case data = <-s.pingCh: + s.pingCheck = false case <-s.closeChan: break - case <-s.pingTimer.C: - logs.Error("mux: ping time out") - s.Close() - break } _ = now.UnmarshalText(data) latency := time.Now().UTC().Sub(now).Seconds() / 2 @@ -222,6 +221,15 @@ func (s *Mux) pingReturn() { } func (s *Mux) readSession() { + go func() { + var connection *conn + for { + connection = s.newConnQueue.Pop() + s.connMap.Set(connection.connId, connection) //it has been set before send ok + s.newConnCh <- connection + s.sendInfo(common.MUX_NEW_CONN_OK, connection.connId, nil) + } + }() go func() { pack := common.MuxPack.Get() var l uint16 @@ -233,18 +241,16 @@ func (s *Mux) readSession() { pack = common.MuxPack.Get() s.bw.StartRead() if l, err = pack.UnPack(s.conn); err != nil { + logs.Error("mux: read session unpack from connection err") + s.Close() break } s.bw.SetCopySize(l) s.pingOk = 0 switch pack.Flag { case common.MUX_NEW_CONN: //new connection - connection := NewConn(pack.Id, s, "npc ") - s.connMap.Set(pack.Id, connection) //it has been set before send ok - //go func(connection *conn) { - s.newConnCh <- connection - s.sendInfo(common.MUX_NEW_CONN_OK, connection.connId, nil) - //}(connection) + connection := NewConn(pack.Id, s) + s.newConnQueue.Push(connection) continue case common.MUX_PING_FLAG: //ping s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, pack.Content) @@ -261,6 +267,7 @@ func (s *Mux) readSession() { case common.MUX_NEW_MSG, common.MUX_NEW_MSG_PART: //new msg from remote connection err = s.newMsg(connection, pack) if err != nil { + logs.Error("mux: read session connection new msg err") connection.Close() } continue diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 4790779..0bfea18 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -33,6 +33,14 @@ func (Self *PriorityQueue) New() { } func (Self *PriorityQueue) Push(packager *common.MuxPackager) { + //logs.Warn("push start") + Self.push(packager) + Self.cond.Broadcast() + //logs.Warn("push finish") + return +} + +func (Self *PriorityQueue) push(packager *common.MuxPackager) { switch packager.Flag { case common.MUX_PING_FLAG, common.MUX_PING_RETURN: Self.highestChain.pushHead(unsafe.Pointer(packager)) @@ -44,8 +52,6 @@ func (Self *PriorityQueue) Push(packager *common.MuxPackager) { default: Self.lowestChain.pushHead(unsafe.Pointer(packager)) } - Self.cond.Signal() - return } const maxStarving uint8 = 8 @@ -121,6 +127,72 @@ func (Self *PriorityQueue) Stop() { Self.cond.Broadcast() } +type ConnQueue struct { + chain *bufChain + starving uint8 + stop bool + cond *sync.Cond +} + +func (Self *ConnQueue) New() { + Self.chain = new(bufChain) + Self.chain.new(32) + locker := new(sync.Mutex) + Self.cond = sync.NewCond(locker) +} + +func (Self *ConnQueue) Push(connection *conn) { + Self.chain.pushHead(unsafe.Pointer(connection)) + Self.cond.Broadcast() + return +} + +func (Self *ConnQueue) Pop() (connection *conn) { + var iter bool + for { + connection = Self.pop() + if connection != nil { + return + } + if Self.stop { + return + } + if iter { + break + // trying to pop twice + } + iter = true + runtime.Gosched() + } + Self.cond.L.Lock() + defer Self.cond.L.Unlock() + for connection = Self.pop(); connection == nil; { + if Self.stop { + return + } + //logs.Warn("queue into wait") + Self.cond.Wait() + // wait for it with no more iter + connection = Self.pop() + //logs.Warn("queue wait finish", packager) + } + return +} + +func (Self *ConnQueue) pop() (connection *conn) { + ptr, ok := Self.chain.popTail() + if ok { + connection = (*conn)(ptr) + return + } + return +} + +func (Self *ConnQueue) Stop() { + Self.stop = true + Self.cond.Broadcast() +} + func NewListElement(buf []byte, l uint16, part bool) (element *common.ListElement, err error) { if uint16(len(buf)) != l { err = errors.New("ListElement: buf length not match") @@ -180,24 +252,12 @@ startPop: if !atomic.CompareAndSwapUint64(&Self.lengthWait, ptrs, Self.chain.head.pack(0, 1)) { goto startPop // another goroutine is pushing } - t := Self.timeout.Sub(time.Now()) - if t <= 0 { - t = time.Minute - } - timer := time.NewTimer(t) - defer timer.Stop() - //logs.Warn("queue into wait") - select { - case <-Self.readOp: - //logs.Warn("queue wait finish") - goto startPop - case <-Self.stopOp: - err = io.EOF - return - case <-timer.C: - err = errors.New("mux.queue: read time out") + err = Self.waitPush() + // there is no more data in queue, wait for it + if err != nil { return } + goto startPop // wait finish, trying to get the new status } // length is not zero, so try to pop for { @@ -223,6 +283,29 @@ func (Self *ReceiveWindowQueue) allowPop() (closed bool) { } } +func (Self *ReceiveWindowQueue) waitPush() (err error) { + //logs.Warn("wait push") + //defer logs.Warn("wait push finish") + t := Self.timeout.Sub(time.Now()) + if t <= 0 { + t = time.Second * 10 + } + timer := time.NewTimer(t) + defer timer.Stop() + //logs.Warn("queue into wait") + select { + case <-Self.readOp: + //logs.Warn("queue wait finish") + return nil + case <-Self.stopOp: + err = io.EOF + return + case <-timer.C: + err = errors.New("mux.queue: read time out") + return + } +} + func (Self *ReceiveWindowQueue) Len() (n uint32) { ptrs := atomic.LoadUint64(&Self.lengthWait) n, _ = Self.chain.head.unpack(ptrs) From aaf79b21f03f04309b1bc10580e4943f2b6b32a0 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Tue, 19 Nov 2019 23:43:52 +0800 Subject: [PATCH 27/34] fine mux connection ping calculation, increase connection waiting time --- README.md | 13 +++++ lib/mux/conn.go | 6 ++- lib/mux/mux.go | 130 +++++++++++++++++++++++++++++++++++++++-------- lib/mux/queue.go | 2 +- 4 files changed, 127 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 4a9ddd4..8988d5a 100644 --- a/README.md +++ b/README.md @@ -824,6 +824,19 @@ nps支持对客户端的隧道数量进行限制，该功能默认是关闭的 nps主要通信默认基于多路复用，无需开启。 +多路复用基于TCP滑动窗口原理设计，动态计算延迟以及带宽来算出应该往网络管道中打入的流量。 +由于主要通信大多采用TCP协议，并无法探测其实时丢包情况，对于产生丢包重传的情况，采用较大的宽容度， +5分钟的等待时间，超时将会关闭当前隧道连接并重新建立，这将会抛弃当前所有的连接。 +在Linux上，可以通过调节内核参数来适应不同应用场景。 + +对于需求大带宽又有一定的丢包的场景，可以保持默认参数不变，尽可能少抛弃连接 +高并发下可根据[Linux系统限制](#Linux系统限制) 调整 + +对于延迟敏感而又有一定丢包的场景，可以适当调整TCP重传次数 +`tcp_syn_retries`, `tcp_retries1`, `tcp_retries2` +高并发同上 +nps会在系统主动关闭连接的时候拿到报错，进而重新建立隧道连接 + ### 环境变量渲染 npc支持环境变量渲染以适应在某些特殊场景下的要求。 diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 23bc5d5..7bb88ae 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -3,6 +3,7 @@ package mux import ( "errors" "io" + "math" "net" "runtime" "sync" @@ -208,7 +209,8 @@ func (Self *ReceiveWindow) calcSize() { // calculating maximum receive window size if Self.count == 0 { //logs.Warn("ping, bw", Self.mux.latency, Self.bw.Get()) - n := uint32(2 * Self.mux.latency * Self.mux.bw.Get() * 1.5 / float64(Self.mux.connMap.Size())) + n := uint32(2 * math.Float64frombits(atomic.LoadUint64(&Self.mux.latency)) * + Self.mux.bw.Get() * 1.5 / float64(Self.mux.connMap.Size())) if n < 8192 { n = 8192 } @@ -471,7 +473,7 @@ start: func (Self *SendWindow) waitReceiveWindow() (err error) { t := Self.timeout.Sub(time.Now()) if t < 0 { - t = time.Minute + t = time.Minute * 5 } timer := time.NewTimer(t) defer timer.Stop() diff --git a/lib/mux/mux.go b/lib/mux/mux.go index b64243f..3872f7e 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -13,21 +13,22 @@ import ( ) type Mux struct { + latency uint64 // we store latency in bits, but it's float64 net.Listener - conn net.Conn - connMap *connMap - newConnCh chan *conn - id int32 - closeChan chan struct{} - IsClose bool - pingOk int - latency float64 - bw *bandwidth - pingCh chan []byte - pingCheck bool - connType string - writeQueue PriorityQueue - newConnQueue ConnQueue + conn net.Conn + connMap *connMap + newConnCh chan *conn + id int32 + closeChan chan struct{} + IsClose bool + pingOk int + counter *latencyCounter + bw *bandwidth + pingCh chan []byte + pingCheckTime uint32 + connType string + writeQueue PriorityQueue + newConnQueue ConnQueue } func NewMux(c net.Conn, connType string) *Mux { @@ -43,6 +44,7 @@ func NewMux(c net.Conn, connType string) *Mux { IsClose: false, connType: connType, pingCh: make(chan []byte), + counter: newLatencyCounter(), } m.writeQueue.New() m.newConnQueue.New() @@ -175,16 +177,16 @@ func (s *Mux) ping() { select { case <-ticker.C: } - if s.pingCheck { + if atomic.LoadUint32(&s.pingCheckTime) >= 60 { logs.Error("mux: ping time out") s.Close() - // more than 5 seconds not receive the ping return package, + // more than 5 minutes not receive the ping return package, // mux conn is damaged, maybe a packet drop, close it break } now, _ := time.Now().UTC().MarshalText() s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) - s.pingCheck = true + atomic.AddUint32(&s.pingCheckTime, 1) if s.pingOk > 10 && s.connType == "kcp" { logs.Error("mux: kcp ping err") s.Close() @@ -205,16 +207,17 @@ func (s *Mux) pingReturn() { } select { case data = <-s.pingCh: - s.pingCheck = false + atomic.StoreUint32(&s.pingCheckTime, 0) case <-s.closeChan: break } _ = now.UnmarshalText(data) latency := time.Now().UTC().Sub(now).Seconds() / 2 - if latency < 0.5 && latency > 0 { - s.latency = latency + if latency > 0 { + atomic.StoreUint64(&s.latency, math.Float64bits(s.counter.Latency(latency))) + // convert float64 to bits, store it atomic } - //logs.Warn("latency", s.latency) + //logs.Warn("latency", math.Float64frombits(atomic.LoadUint64(&s.latency))) common.WindowBuff.Put(data) } }() @@ -379,3 +382,88 @@ func (Self *bandwidth) Get() (bw float64) { } return Self.readBandwidth } + +const counterBits = 4 +const counterMask = 1<> counterBits) & counterMask) + // we set head is 4 bits + min = uint8(idxs & counterMask) + return +} + +func (Self *latencyCounter) pack(head, min uint8) uint8 { + return uint8(head< value { + min = head + } + head++ + Self.headMin = Self.pack(head, min) +} + +func (Self *latencyCounter) minimal() (min uint8) { + var val float64 + var i uint8 + for i = 0; i < counterMask; i++ { + if Self.buf[i] > 0 { + if val > Self.buf[i] { + val = Self.buf[i] + min = i + } + } + } + return +} + +func (Self *latencyCounter) Latency(value float64) (latency float64) { + Self.add(value) + _, min := Self.unpack(Self.headMin) + latency = Self.buf[min] * Self.countSuccess() + return +} + +const lossRatio = 1.6 + +func (Self *latencyCounter) countSuccess() (successRate float64) { + var success, loss, i uint8 + _, min := Self.unpack(Self.headMin) + for i = 0; i < counterMask; i++ { + if Self.buf[i] > lossRatio*Self.buf[min] && Self.buf[i] > 0 { + loss++ + } + if Self.buf[i] <= lossRatio*Self.buf[min] && Self.buf[i] > 0 { + success++ + } + } + // counting all the data in the ring buf, except zero + successRate = float64(success) / float64(loss+success) + return +} diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 0bfea18..2fe8a44 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -288,7 +288,7 @@ func (Self *ReceiveWindowQueue) waitPush() (err error) { //defer logs.Warn("wait push finish") t := Self.timeout.Sub(time.Now()) if t <= 0 { - t = time.Second * 10 + t = time.Minute * 5 } timer := time.NewTimer(t) defer timer.Stop() From bfe08e5114e320fef84d67a6382a358f8254fe1f Mon Sep 17 00:00:00 2001 From: zhangzc <50092028+43280398@users.noreply.github.com> Date: Wed, 20 Nov 2019 11:47:55 +0800 Subject: [PATCH 28/34] Update util.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修改nginx代理转发后无法获取真实ip --- lib/common/util.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/common/util.go b/lib/common/util.go index 1f54a6f..9a60846 100755 --- a/lib/common/util.go +++ b/lib/common/util.go @@ -108,6 +108,9 @@ func ChangeHostAndHeader(r *http.Request, host string, header string, addr strin } } addr = strings.Split(addr, ":")[0] + if prior, ok := r.Header["X-Forwarded-For"]; ok { + addr = strings.Join(prior, ", ") + ", " + addr + } r.Header.Set("X-Forwarded-For", addr) r.Header.Set("X-Real-IP", addr) } From 9bb8230fc17e59101d397c16aef4562a481020a8 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Thu, 21 Nov 2019 23:53:06 +0800 Subject: [PATCH 29/34] fix several race condition, change slide window max size 2G to 32M, add buffer release --- lib/common/const.go | 3 ++- lib/mux/conn.go | 25 +++++++++++++++++- lib/mux/map.go | 63 +++++++++++++++++++++++---------------------- lib/mux/mux.go | 57 +++++++++++++++++++++++++++++++++------- lib/mux/queue.go | 40 ++++++++++++++++------------ 5 files changed, 129 insertions(+), 59 deletions(-) diff --git a/lib/common/const.go b/lib/common/const.go index f57ce4f..2fd5bb6 100644 --- a/lib/common/const.go +++ b/lib/common/const.go @@ -49,5 +49,6 @@ const ( MUX_PING_RETURN MUX_PING int32 = -1 MAXIMUM_SEGMENT_SIZE = PoolSizeWindow - MAXIMUM_WINDOW_SIZE = 1<<31 - 1 + MAXIMUM_WINDOW_SIZE = 1 << 25 // 1<<31-1 TCP slide window size is very large, + // we use 32M, reduce memory usage ) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 7bb88ae..c94ab4d 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -210,7 +210,7 @@ func (Self *ReceiveWindow) calcSize() { if Self.count == 0 { //logs.Warn("ping, bw", Self.mux.latency, Self.bw.Get()) n := uint32(2 * math.Float64frombits(atomic.LoadUint64(&Self.mux.latency)) * - Self.mux.bw.Get() * 1.5 / float64(Self.mux.connMap.Size())) + Self.mux.bw.Get() / float64(Self.mux.connMap.Size())) if n < 8192 { n = 8192 } @@ -279,6 +279,9 @@ copyData: // on the first Read method invoked, Self.off and Self.element.l // both zero value common.ListElementPool.Put(Self.element) + if Self.closeOp { + return 0, io.EOF + } Self.element, err = Self.bufQueue.Pop() // if the queue is empty, Pop method will wait until one element push // into the queue successful, or timeout. @@ -343,6 +346,26 @@ func (Self *ReceiveWindow) Stop() { func (Self *ReceiveWindow) CloseWindow() { Self.window.CloseWindow() Self.Stop() + Self.release() +} + +func (Self *ReceiveWindow) release() { + //if Self.element != nil { + // if Self.element.Buf != nil { + // common.WindowBuff.Put(Self.element.Buf) + // } + // common.ListElementPool.Put(Self.element) + //} + for { + Self.element = Self.bufQueue.TryPop() + if Self.element == nil { + return + } + if Self.element.Buf != nil { + common.WindowBuff.Put(Self.element.Buf) + } + common.ListElementPool.Put(Self.element) + } // release resource } type SendWindow struct { diff --git a/lib/mux/map.go b/lib/mux/map.go index 8f07dee..86d09b5 100644 --- a/lib/mux/map.go +++ b/lib/mux/map.go @@ -2,32 +2,35 @@ package mux import ( "sync" - "time" ) type connMap struct { connMap map[int32]*conn - closeCh chan struct{} + //closeCh chan struct{} sync.RWMutex } func NewConnMap() *connMap { connMap := &connMap{ connMap: make(map[int32]*conn), - closeCh: make(chan struct{}), + //closeCh: make(chan struct{}), } - go connMap.clean() + //go connMap.clean() return connMap } func (s *connMap) Size() (n int) { - return len(s.connMap) + s.Lock() + n = len(s.connMap) + s.Unlock() + return } func (s *connMap) Get(id int32) (*conn, bool) { s.Lock() - defer s.Unlock() - if v, ok := s.connMap[id]; ok && v != nil { + v, ok := s.connMap[id] + s.Unlock() + if ok && v != nil { return v, true } return nil, false @@ -35,40 +38,38 @@ func (s *connMap) Get(id int32) (*conn, bool) { func (s *connMap) Set(id int32, v *conn) { s.Lock() - defer s.Unlock() s.connMap[id] = v + s.Unlock() } func (s *connMap) Close() { - s.Lock() - defer s.Unlock() + //s.closeCh <- struct{}{} // stop the clean goroutine first for _, v := range s.connMap { - v.isClose = true + v.Close() // close all the connections in the mux } - s.closeCh <- struct{}{} } func (s *connMap) Delete(id int32) { s.Lock() - defer s.Unlock() delete(s.connMap, id) + s.Unlock() } -func (s *connMap) clean() { - ticker := time.NewTimer(time.Minute * 1) - for { - select { - case <-ticker.C: - s.Lock() - for _, v := range s.connMap { - if v.isClose { - delete(s.connMap, v.connId) - } - } - s.Unlock() - case <-s.closeCh: - ticker.Stop() - return - } - } -} +//func (s *connMap) clean() { +// ticker := time.NewTimer(time.Minute * 1) +// for { +// select { +// case <-ticker.C: +// s.Lock() +// for _, v := range s.connMap { +// if v.isClose { +// delete(s.connMap, v.connId) +// } +// } +// s.Unlock() +// case <-s.closeCh: +// ticker.Stop() +// return +// } +// } +//} diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 3872f7e..8c4febb 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -122,6 +122,9 @@ func (s *Mux) packBuf() { } buffer.Reset() pack := s.writeQueue.Pop() + if s.IsClose { + break + } //buffer := common.BuffPool.Get() err := pack.Pack(buffer) common.MuxPack.Put(pack) @@ -218,7 +221,9 @@ func (s *Mux) pingReturn() { // convert float64 to bits, store it atomic } //logs.Warn("latency", math.Float64frombits(atomic.LoadUint64(&s.latency))) - common.WindowBuff.Put(data) + if cap(data) > 0 { + common.WindowBuff.Put(data) + } } }() } @@ -227,7 +232,13 @@ func (s *Mux) readSession() { go func() { var connection *conn for { + if s.IsClose { + break + } connection = s.newConnQueue.Pop() + if s.IsClose { + break // make sure that is closed + } s.connMap.Set(connection.connId, connection) //it has been set before send ok s.newConnCh <- connection s.sendInfo(common.MUX_NEW_CONN_OK, connection.connId, nil) @@ -287,9 +298,9 @@ func (s *Mux) readSession() { connection.sendWindow.SetSize(pack.Window, pack.ReadLength) continue case common.MUX_CONN_CLOSE: //close the connection - s.connMap.Delete(pack.Id) - //go func(connection *conn) { connection.closeFlag = true + //s.connMap.Delete(pack.Id) + //go func(connection *conn) { connection.receiveWindow.Stop() // close signal to receive window //}(connection) continue @@ -322,17 +333,42 @@ func (s *Mux) newMsg(connection *conn, pack *common.MuxPackager) (err error) { return } -func (s *Mux) Close() error { +func (s *Mux) Close() (err error) { logs.Warn("close mux") if s.IsClose { return errors.New("the mux has closed") } s.IsClose = true s.connMap.Close() + s.connMap = nil //s.bufQueue.Stop() s.closeChan <- struct{}{} close(s.newConnCh) - return s.conn.Close() + err = s.conn.Close() + s.release() + return +} + +func (s *Mux) release() { + for { + pack := s.writeQueue.TryPop() + if pack == nil { + break + } + if pack.BasePackager.Content != nil { + common.WindowBuff.Put(pack.BasePackager.Content) + } + common.MuxPack.Put(pack) + } + for { + connection := s.newConnQueue.TryPop() + if connection == nil { + break + } + connection = nil + } + s.writeQueue.Stop() + s.newConnQueue.Stop() } //get new connId as unique flag @@ -352,7 +388,7 @@ type bandwidth struct { readStart time.Time lastReadStart time.Time bufLength uint16 - readBandwidth float64 + readBandwidth uint64 // store in bits, but it's float64 } func (Self *bandwidth) StartRead() { @@ -371,16 +407,17 @@ func (Self *bandwidth) SetCopySize(n uint16) { func (Self *bandwidth) calcBandWidth() { t := Self.readStart.Sub(Self.lastReadStart) - Self.readBandwidth = float64(Self.bufLength) / t.Seconds() + atomic.StoreUint64(&Self.readBandwidth, math.Float64bits(float64(Self.bufLength)/t.Seconds())) Self.bufLength = 0 } func (Self *bandwidth) Get() (bw float64) { // The zero value, 0 for numeric types - if Self.readBandwidth <= 0 { - Self.readBandwidth = 100 + bw = math.Float64frombits(atomic.LoadUint64(&Self.readBandwidth)) + if bw <= 0 { + bw = 100 } - return Self.readBandwidth + return } const counterBits = 4 diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 2fe8a44..2288e40 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -59,7 +59,7 @@ const maxStarving uint8 = 8 func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { var iter bool for { - packager = Self.pop() + packager = Self.TryPop() if packager != nil { return } @@ -75,20 +75,20 @@ func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { } Self.cond.L.Lock() defer Self.cond.L.Unlock() - for packager = Self.pop(); packager == nil; { + for packager = Self.TryPop(); packager == nil; { if Self.stop { return } //logs.Warn("queue into wait") Self.cond.Wait() // wait for it with no more iter - packager = Self.pop() + packager = Self.TryPop() //logs.Warn("queue wait finish", packager) } return } -func (Self *PriorityQueue) pop() (packager *common.MuxPackager) { +func (Self *PriorityQueue) TryPop() (packager *common.MuxPackager) { ptr, ok := Self.highestChain.popTail() if ok { packager = (*common.MuxPackager)(ptr) @@ -150,7 +150,7 @@ func (Self *ConnQueue) Push(connection *conn) { func (Self *ConnQueue) Pop() (connection *conn) { var iter bool for { - connection = Self.pop() + connection = Self.TryPop() if connection != nil { return } @@ -166,20 +166,20 @@ func (Self *ConnQueue) Pop() (connection *conn) { } Self.cond.L.Lock() defer Self.cond.L.Unlock() - for connection = Self.pop(); connection == nil; { + for connection = Self.TryPop(); connection == nil; { if Self.stop { return } //logs.Warn("queue into wait") Self.cond.Wait() // wait for it with no more iter - connection = Self.pop() + connection = Self.TryPop() //logs.Warn("queue wait finish", packager) } return } -func (Self *ConnQueue) pop() (connection *conn) { +func (Self *ConnQueue) TryPop() (connection *conn) { ptr, ok := Self.chain.popTail() if ok { connection = (*conn)(ptr) @@ -261,18 +261,26 @@ startPop: } // length is not zero, so try to pop for { - ptr, ok := Self.chain.popTail() - if ok { - //logs.Warn("window pop before", Self.Len()) - element = (*common.ListElement)(ptr) - atomic.AddUint64(&Self.lengthWait, ^(uint64(element.L)< Date: Sat, 23 Nov 2019 00:42:46 +0800 Subject: [PATCH 30/34] change some bandwidth calculation --- lib/common/pool.go | 2 +- lib/mux/conn.go | 22 +++++++++++++--------- lib/mux/mux.go | 32 +++++++++++++++++--------------- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/lib/common/pool.go b/lib/common/pool.go index 5d4da00..1f7a47e 100644 --- a/lib/common/pool.go +++ b/lib/common/pool.go @@ -10,7 +10,7 @@ const PoolSizeSmall = 100 const PoolSizeUdp = 1472 const PoolSizeCopy = 32 << 10 const PoolSizeBuffer = 4096 -const PoolSizeWindow = PoolSizeBuffer - 16 - 32 - 32 - 8 +const PoolSizeWindow = PoolSizeBuffer - 2 - 4 - 4 - 1 var BufPool = sync.Pool{ New: func() interface{} { diff --git a/lib/mux/conn.go b/lib/mux/conn.go index c94ab4d..1b7a920 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -190,7 +190,7 @@ func (Self *ReceiveWindow) New(mux *Mux) { // initial a window for receive Self.bufQueue.New() Self.element = common.ListElementPool.Get() - Self.maxSize = 4096 + Self.maxSize = common.MAXIMUM_SEGMENT_SIZE Self.mux = mux Self.window.New() } @@ -209,21 +209,25 @@ func (Self *ReceiveWindow) calcSize() { // calculating maximum receive window size if Self.count == 0 { //logs.Warn("ping, bw", Self.mux.latency, Self.bw.Get()) - n := uint32(2 * math.Float64frombits(atomic.LoadUint64(&Self.mux.latency)) * - Self.mux.bw.Get() / float64(Self.mux.connMap.Size())) - if n < 8192 { - n = 8192 + conns := Self.mux.connMap.Size() + n := uint32(math.Float64frombits(atomic.LoadUint64(&Self.mux.latency)) * + Self.mux.bw.Get() / float64(conns)) + if n < common.MAXIMUM_SEGMENT_SIZE*2 { + n = common.MAXIMUM_SEGMENT_SIZE * 2 } bufLen := Self.bufQueue.Len() if n < bufLen { n = bufLen } + if n < Self.maxSize/2 { + n = Self.maxSize / 2 + } // set the minimal size if n > 2*Self.maxSize { n = 2 * Self.maxSize } - if n > common.MAXIMUM_WINDOW_SIZE { - n = common.MAXIMUM_WINDOW_SIZE + if n > (common.MAXIMUM_WINDOW_SIZE / uint32(conns)) { + n = common.MAXIMUM_WINDOW_SIZE / uint32(conns) } // set the maximum size //logs.Warn("n", n) @@ -377,8 +381,8 @@ type SendWindow struct { func (Self *SendWindow) New(mux *Mux) { Self.setSizeCh = make(chan struct{}) - Self.maxSize = 4096 - atomic.AddUint64(&Self.remainingWait, uint64(4096)<= 16384 { + if Self.bufLength >= 3072000 { Self.lastReadStart, Self.readStart = Self.readStart, time.Now() Self.calcBandWidth() } } func (Self *bandwidth) SetCopySize(n uint16) { - Self.bufLength += n + Self.bufLength += uint32(n) } func (Self *bandwidth) calcBandWidth() { @@ -417,6 +418,7 @@ func (Self *bandwidth) Get() (bw float64) { if bw <= 0 { bw = 100 } + //logs.Warn(bw) return } From 32e3d411ad1d5287433e43becb9827e07324fcb9 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sun, 24 Nov 2019 21:19:25 +0800 Subject: [PATCH 31/34] change initial window size --- lib/mux/conn.go | 10 +++++----- lib/mux/mux.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 1b7a920..f665248 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -190,7 +190,7 @@ func (Self *ReceiveWindow) New(mux *Mux) { // initial a window for receive Self.bufQueue.New() Self.element = common.ListElementPool.Get() - Self.maxSize = common.MAXIMUM_SEGMENT_SIZE + Self.maxSize = common.MAXIMUM_SEGMENT_SIZE * 10 Self.mux = mux Self.window.New() } @@ -212,8 +212,8 @@ func (Self *ReceiveWindow) calcSize() { conns := Self.mux.connMap.Size() n := uint32(math.Float64frombits(atomic.LoadUint64(&Self.mux.latency)) * Self.mux.bw.Get() / float64(conns)) - if n < common.MAXIMUM_SEGMENT_SIZE*2 { - n = common.MAXIMUM_SEGMENT_SIZE * 2 + if n < common.MAXIMUM_SEGMENT_SIZE*10 { + n = common.MAXIMUM_SEGMENT_SIZE * 10 } bufLen := Self.bufQueue.Len() if n < bufLen { @@ -381,8 +381,8 @@ type SendWindow struct { func (Self *SendWindow) New(mux *Mux) { Self.setSizeCh = make(chan struct{}) - Self.maxSize = common.MAXIMUM_SEGMENT_SIZE - atomic.AddUint64(&Self.remainingWait, uint64(common.MAXIMUM_SEGMENT_SIZE)<= 3072000 { + if Self.bufLength >= common.MAXIMUM_SEGMENT_SIZE*300 { Self.lastReadStart, Self.readStart = Self.readStart, time.Now() Self.calcBandWidth() } From 78ebeba1bbd47ebf1f6cce7985d38b8e1041c5f9 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Wed, 27 Nov 2019 22:46:34 +0800 Subject: [PATCH 32/34] minor bug fix, docker --- Dockerfile.npc | 10 ++++++++++ Dockerfile.nps | 11 +++++++++++ README.md | 7 ++++++- lib/conn/listener.go | 7 +++++++ lib/mux/mux.go | 14 +++++++------- lib/mux/queue.go | 2 +- 6 files changed, 42 insertions(+), 9 deletions(-) create mode 100755 Dockerfile.npc create mode 100755 Dockerfile.nps diff --git a/Dockerfile.npc b/Dockerfile.npc new file mode 100755 index 0000000..ae6715a --- /dev/null +++ b/Dockerfile.npc @@ -0,0 +1,10 @@ +FROM golang as builder +WORKDIR /go/src/github.com/cnlh/nps +COPY . . +RUN go get -d -v ./... +RUN CGO_ENABLED=0 go build -ldflags="-w -s -extldflags -static" ./cmd/npc/npc.go + +FROM scratch +COPY --from=builder /go/src/github.com/cnlh/nps/npc / +VOLUME /conf +ENTRYPOINT ["/npc"] diff --git a/Dockerfile.nps b/Dockerfile.nps new file mode 100755 index 0000000..698ced9 --- /dev/null +++ b/Dockerfile.nps @@ -0,0 +1,11 @@ +FROM golang as builder +WORKDIR /go/src/github.com/cnlh/nps +COPY . . +RUN go get -d -v ./... +RUN CGO_ENABLED=0 go build -ldflags="-w -s -extldflags -static" ./cmd/nps/nps.go + +FROM scratch +COPY --from=builder /go/src/github.com/cnlh/nps/nps / +COPY --from=builder /go/src/github.com/cnlh/nps/web /web +VOLUME /conf +CMD ["/nps"] diff --git a/README.md b/README.md index 8988d5a..ba93cb5 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ nps是一款轻量级、高性能、功能强大的**内网穿透**代理服务 * [安装](#安装) * [编译安装](#源码安装) * [release安装](#release安装) + * [docker安装](#docker安装) * [使用示例（以web主控模式为主）](#使用示例) * [统一准备工作](#统一准备工作(必做)) * [http|https域名解析](#域名解析) @@ -121,7 +122,7 @@ nps是一款轻量级、高性能、功能强大的**内网穿透**代理服务 ## 安装 -### releases安装 +### release安装 > [releases](https://github.com/cnlh/nps/releases) 下载对应的系统版本即可，服务端和客户端是单独的 @@ -134,6 +135,10 @@ nps是一款轻量级、高性能、功能强大的**内网穿透**代理服务 > go build cmd/npc/npc.go +### docker安装 +> [server](https://hub.docker.com/r/ffdfgdfg/nps) +> [client](https://hub.docker.com/r/ffdfgdfg/npc) + ## 使用示例 ### 统一准备工作（必做） diff --git a/lib/conn/listener.go b/lib/conn/listener.go index f80e01d..bd8e443 100644 --- a/lib/conn/listener.go +++ b/lib/conn/listener.go @@ -43,9 +43,16 @@ func Accept(l net.Listener, f func(c net.Conn)) { if strings.Contains(err.Error(), "use of closed network connection") { break } + if strings.Contains(err.Error(), "the mux has closed") { + break + } logs.Warn(err) continue } + if c == nil { + logs.Warn("nil connection") + break + } go f(c) } } diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 6c9c7fd..a43510a 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -21,7 +21,7 @@ type Mux struct { id int32 closeChan chan struct{} IsClose bool - pingOk int + pingOk uint32 counter *latencyCounter bw *bandwidth pingCh chan []byte @@ -101,7 +101,7 @@ func (s *Mux) sendInfo(flag uint8, id int32, data ...interface{}) { err = pack.NewPac(flag, id, data...) if err != nil { common.MuxPack.Put(pack) - logs.Error("mux: new pack err") + logs.Error("mux: new pack err", err) s.Close() return } @@ -191,12 +191,12 @@ func (s *Mux) ping() { now, _ := time.Now().UTC().MarshalText() s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) atomic.AddUint32(&s.pingCheckTime, 1) - if s.pingOk > 10 && s.connType == "kcp" { + if atomic.LoadUint32(&s.pingOk) > 10 && s.connType == "kcp" { logs.Error("mux: kcp ping err") s.Close() break } - s.pingOk++ + atomic.AddUint32(&s.pingOk, 1) } }() } @@ -256,12 +256,12 @@ func (s *Mux) readSession() { pack = common.MuxPack.Get() s.bw.StartRead() if l, err = pack.UnPack(s.conn); err != nil { - logs.Error("mux: read session unpack from connection err") + logs.Error("mux: read session unpack from connection err", err) s.Close() break } s.bw.SetCopySize(l) - s.pingOk = 0 + atomic.StoreUint32(&s.pingOk, 0) switch pack.Flag { case common.MUX_NEW_CONN: //new connection connection := NewConn(pack.Id, s) @@ -282,7 +282,7 @@ func (s *Mux) readSession() { case common.MUX_NEW_MSG, common.MUX_NEW_MSG_PART: //new msg from remote connection err = s.newMsg(connection, pack) if err != nil { - logs.Error("mux: read session connection new msg err") + logs.Error("mux: read session connection new msg err", err) connection.Close() } continue diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 2288e40..212563c 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -209,10 +209,10 @@ func NewListElement(buf []byte, l uint16, part bool) (element *common.ListElemen } type ReceiveWindowQueue struct { + lengthWait uint64 chain *bufChain stopOp chan struct{} readOp chan struct{} - lengthWait uint64 timeout time.Time } From f352ee8f398bff2f9367acc65c5a6cc95e89d101 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Thu, 28 Nov 2019 21:21:16 +0800 Subject: [PATCH 33/34] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 38 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 ++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..041b570 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Opening '...' +2. Click on '....' +3. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots or logs** +Add screenshots or logs to help explain your problem. + +**Server (please complete the following information):** + - OS: [e.g. Centos, Windows] + - ARCH: [e.g. Amd64, Arm] + - Tunnel [e.g. TCP, HTTP] + - Version [e.g. 0.24.0] + +**Client (please complete the following information):** + - OS: [e.g. Centos, Windows] + - ARCH: [e.g. Amd64, Arm] + - Tunnel [e.g. TCP, HTTP] + - Version [e.g. 0.24.0] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..11fc491 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. From e429b17e63c42b9db64c05e8d363ebf2fb6f37d3 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Fri, 29 Nov 2019 00:23:55 +0800 Subject: [PATCH 34/34] strange 64bit alignment --- lib/mux/queue.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 212563c..6f14faa 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -209,10 +209,15 @@ func NewListElement(buf []byte, l uint16, part bool) (element *common.ListElemen } type ReceiveWindowQueue struct { - lengthWait uint64 chain *bufChain stopOp chan struct{} readOp chan struct{} + lengthWait uint64 // really strange ???? need put here + // https://golang.org/pkg/sync/atomic/#pkg-note-BUG + // On non-Linux ARM, the 64-bit functions use instructions unavailable before the ARMv6k core. + // On ARM, x86-32, and 32-bit MIPS, it is the caller's responsibility + // to arrange for 64-bit alignment of 64-bit words accessed atomically. + // The first word in a variable or in an allocated struct, array, or slice can be relied upon to be 64-bit aligned. timeout time.Time }