From 00900c13156af77cd8919168f67ad8662888c077 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Fri, 23 Aug 2019 18:53:36 +0800 Subject: [PATCH 01/38] mux test --- lib/common/const.go | 12 +++ lib/common/netpackager.go | 175 ++++++++++++++++++++++++++++++++++++ lib/mux/bytes.go | 2 +- lib/mux/conn.go | 42 ++++----- lib/mux/mux.go | 185 ++++++++++++++++++-------------------- lib/mux/mux_test.go | 4 +- lib/pool/pool.go | 32 ++++++- 7 files changed, 324 insertions(+), 128 deletions(-) create mode 100644 lib/common/netpackager.go diff --git a/lib/common/const.go b/lib/common/const.go index ffb2fa6..d77f16b 100644 --- a/lib/common/const.go +++ b/lib/common/const.go @@ -36,3 +36,15 @@ WWW-Authenticate: Basic realm="easyProxy" ` ) + +const ( + MUX_PING_FLAG uint8 = iota + MUX_NEW_CONN_OK + MUX_NEW_CONN_Fail + MUX_NEW_MSG + MUX_MSG_SEND_OK + MUX_NEW_CONN + MUX_CONN_CLOSE + MUX_PING_RETURN + MUX_PING int32 = -1 +) diff --git a/lib/common/netpackager.go b/lib/common/netpackager.go new file mode 100644 index 0000000..315f645 --- /dev/null +++ b/lib/common/netpackager.go @@ -0,0 +1,175 @@ +package common + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "github.com/cnlh/nps/lib/pool" + "io" + "strings" +) + +type NetPackager interface { + Pack(writer io.Writer) (err error) + UnPack(reader io.Reader) (err error) +} + +type BasePackager struct { + Length uint32 + Content []byte +} + +func (Self *BasePackager) NewPac(contents ...interface{}) (err error) { + Self.clean() + for _, content := range contents { + switch content.(type) { + case nil: + Self.Content = Self.Content[:0] + case []byte: + Self.Content = append(Self.Content, content.([]byte)...) + case string: + Self.Content = append(Self.Content, []byte(content.(string))...) + Self.Content = append(Self.Content, []byte(CONN_DATA_SEQ)...) + default: + err = Self.marshal(content) + } + } + Self.setLength() + return +} + +//似乎这里涉及到父类作用域问题，当子类调用父类的方法时，其struct仅仅为父类的 +func (Self *BasePackager) Pack(writer io.Writer) (err error) { + err = binary.Write(writer, binary.LittleEndian, Self.Length) + if err != nil { + return + } + err = binary.Write(writer, binary.LittleEndian, Self.Content) + //logs.Warn(Self.Length, string(Self.Content)) + return +} + +//Unpack 会导致传入的数字类型转化成float64！！ +//主要原因是json unmarshal并未传入正确的数据类型 +func (Self *BasePackager) UnPack(reader io.Reader) (err error) { + err = binary.Read(reader, binary.LittleEndian, &Self.Length) + if err != nil { + return + } + Self.Content = pool.GetBufPoolCopy() + Self.Content = Self.Content[:Self.Length] + //n, err := io.ReadFull(reader, Self.Content) + //if n != int(Self.Length) { + // err = io.ErrUnexpectedEOF + //} + err = binary.Read(reader, binary.LittleEndian, &Self.Content) + return +} + +func (Self *BasePackager) marshal(content interface{}) (err error) { + tmp, err := json.Marshal(content) + if err != nil { + return err + } + Self.Content = append(Self.Content, tmp...) + return +} + +func (Self *BasePackager) Unmarshal(content interface{}) (err error) { + err = json.Unmarshal(Self.Content, content) + if err != nil { + return err + } + return +} + +func (Self *BasePackager) setLength() { + Self.Length = uint32(len(Self.Content)) + return +} + +func (Self *BasePackager) clean() { + Self.Length = 0 + Self.Content = Self.Content[:0] +} + +func (Self *BasePackager) Split() (strList []string) { + n := bytes.IndexByte(Self.Content, 0) + strList = strings.Split(string(Self.Content[:n]), CONN_DATA_SEQ) + strList = strList[0 : len(strList)-1] + return +} + +type ConnPackager struct { // Todo + ConnType uint8 + BasePackager +} + +func (Self *ConnPackager) NewPac(connType uint8, content ...interface{}) (err error) { + Self.ConnType = connType + err = Self.BasePackager.NewPac(content...) + return +} + +func (Self *ConnPackager) Pack(writer io.Writer) (err error) { + err = binary.Write(writer, binary.LittleEndian, Self.ConnType) + if err != nil { + return + } + err = Self.BasePackager.Pack(writer) + return +} + +func (Self *ConnPackager) UnPack(reader io.Reader) (err error) { + err = binary.Read(reader, binary.LittleEndian, &Self.ConnType) + if err != nil && err != io.EOF { + return + } + err = Self.BasePackager.UnPack(reader) + return +} + +type MuxPackager struct { + Flag uint8 + Id int32 + BasePackager +} + +func (Self *MuxPackager) NewPac(flag uint8, id int32, content ...interface{}) (err error) { + Self.Flag = flag + Self.Id = id + if flag == MUX_NEW_MSG { + err = Self.BasePackager.NewPac(content...) + } + return +} + +func (Self *MuxPackager) Pack(writer io.Writer) (err error) { + err = binary.Write(writer, binary.LittleEndian, Self.Flag) + if err != nil { + return + } + err = binary.Write(writer, binary.LittleEndian, Self.Id) + if err != nil { + return + } + if Self.Flag == MUX_NEW_MSG { + err = Self.BasePackager.Pack(writer) + } + return +} + +func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { + err = binary.Read(reader, binary.LittleEndian, &Self.Flag) + if err != nil { + return + } + err = binary.Read(reader, binary.LittleEndian, &Self.Id) + if err != nil { + return + } + if Self.Flag == MUX_NEW_MSG { + err = Self.BasePackager.UnPack(reader) + } + return +} diff --git a/lib/mux/bytes.go b/lib/mux/bytes.go index a7e17f7..c44bad4 100644 --- a/lib/mux/bytes.go +++ b/lib/mux/bytes.go @@ -20,7 +20,7 @@ func WriteLenBytes(buf []byte, w io.Writer) (int, error) { //read bytes by length func ReadLenBytes(buf []byte, r io.Reader) (int, error) { - var l int32 + var l uint32 var err error if binary.Read(r, binary.LittleEndian, &l) != nil { return 0, err diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 9e66577..a14e98d 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -2,10 +2,11 @@ package mux import ( "errors" + "github.com/cnlh/nps/lib/common" "github.com/cnlh/nps/lib/pool" + "github.com/cnlh/nps/vender/github.com/astaxie/beego/logs" "io" "net" - "sync" "time" ) @@ -30,8 +31,6 @@ type conn struct { mux *Mux } -var connPool = sync.Pool{} - func NewConn(connId int32, mux *Mux) *conn { c := &conn{ readCh: make(chan struct{}), @@ -73,9 +72,15 @@ func (s *conn) Read(buf []byte) (n int, err error) { return 0, io.EOF } else { pool.PutBufPoolCopy(s.readBuffer) - s.readBuffer = node.val - s.endRead = node.l - s.startRead = 0 + if node.val == nil { + //close + s.Close() + logs.Warn("close from read msg ", s.connId) + } else { + s.readBuffer = node.val + s.endRead = node.l + s.startRead = 0 + } } } if len(buf) < s.endRead-s.startRead { @@ -84,12 +89,11 @@ func (s *conn) Read(buf []byte) (n int, err error) { } else { n = copy(buf, s.readBuffer[s.startRead:s.endRead]) s.startRead += n - s.mux.sendInfo(MUX_MSG_SEND_OK, s.connId, nil) } return } -func (s *conn) Write(buf []byte) (int, error) { +func (s *conn) Write(buf []byte) (n int, err error) { if s.isClose { return 0, errors.New("the conn has closed") } @@ -115,15 +119,11 @@ func (s *conn) write(buf []byte, ch chan struct{}) { start := 0 l := len(buf) for { - if s.hasWrite > 50 { - <-s.getStatusCh - } - s.hasWrite++ if l-start > pool.PoolSizeCopy { - s.mux.sendInfo(MUX_NEW_MSG, s.connId, buf[start:start+pool.PoolSizeCopy]) + s.mux.sendInfo(common.MUX_NEW_MSG, s.connId, buf[start:start+pool.PoolSizeCopy]) start += pool.PoolSizeCopy } else { - s.mux.sendInfo(MUX_NEW_MSG, s.connId, buf[start:l]) + s.mux.sendInfo(common.MUX_NEW_MSG, s.connId, buf[start:l]) break } } @@ -132,16 +132,7 @@ func (s *conn) write(buf []byte, ch chan struct{}) { func (s *conn) Close() error { if s.isClose { - return errors.New("the conn has closed") - } - times := 0 -retry: - if s.waitQueue.Size() > 0 && times < 600 { - time.Sleep(time.Millisecond * 100) - times++ - goto retry - } - if s.isClose { + logs.Warn("already closed", s.connId) return errors.New("the conn has closed") } s.isClose = true @@ -152,9 +143,8 @@ retry: s.waitQueue.Clear() s.mux.connMap.Delete(s.connId) if !s.mux.IsClose { - s.mux.sendInfo(MUX_CONN_CLOSE, s.connId, nil) + s.mux.sendInfo(common.MUX_CONN_CLOSE, s.connId, nil) } - connPool.Put(s) return nil } diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 315bc68..bfd82ff 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -1,10 +1,10 @@ package mux import ( - "bytes" - "encoding/binary" "errors" + "github.com/cnlh/nps/lib/common" "github.com/cnlh/nps/lib/pool" + "github.com/cnlh/nps/vender/github.com/astaxie/beego/logs" "math" "net" "sync" @@ -12,40 +12,30 @@ import ( "time" ) -const ( - MUX_PING_FLAG int32 = iota - MUX_NEW_CONN_OK - MUX_NEW_CONN_Fail - MUX_NEW_MSG - MUX_MSG_SEND_OK - MUX_NEW_CONN - MUX_PING - MUX_CONN_CLOSE - MUX_PING_RETURN -) - type Mux struct { net.Listener - conn net.Conn - connMap *connMap - newConnCh chan *conn - id int32 - closeChan chan struct{} - IsClose bool - pingOk int - connType string + conn net.Conn + connMap *connMap + newConnCh chan *conn + id int32 + closeChan chan struct{} + IsClose bool + pingOk int + connType string + writeQueue *sliceEntry sync.Mutex } func NewMux(c net.Conn, connType string) *Mux { m := &Mux{ - conn: c, - connMap: NewConnMap(), - id: 0, - closeChan: make(chan struct{}), - newConnCh: make(chan *conn), - IsClose: false, - connType: connType, + conn: c, + connMap: NewConnMap(), + id: 0, + closeChan: make(chan struct{}), + newConnCh: make(chan *conn), + IsClose: false, + connType: connType, + writeQueue: NewQueue(), } //read session by flag go m.readSession() @@ -61,7 +51,7 @@ func (s *Mux) NewConn() (*conn, error) { conn := NewConn(s.getId(), s) //it must be set before send s.connMap.Set(conn.connId, conn) - if err := s.sendInfo(MUX_NEW_CONN, conn.connId, nil); err != nil { + if err := s.sendInfo(common.MUX_NEW_CONN, conn.connId, nil); err != nil { return nil, err } //set a timer timeout 30 second @@ -91,19 +81,28 @@ func (s *Mux) Addr() net.Addr { return s.conn.LocalAddr() } -func (s *Mux) sendInfo(flag int32, id int32, content []byte) error { - raw := bytes.NewBuffer([]byte{}) - binary.Write(raw, binary.LittleEndian, flag) - binary.Write(raw, binary.LittleEndian, id) - if content != nil && len(content) > 0 { - binary.Write(raw, binary.LittleEndian, int32(len(content))) - binary.Write(raw, binary.LittleEndian, content) - } - if _, err := s.conn.Write(raw.Bytes()); err != nil { +func (s *Mux) sendInfo(flag uint8, id int32, content []byte) (err error) { + buf := pool.BuffPool.Get() + defer pool.BuffPool.Put(buf) + pack := common.MuxPackager{} + err = pack.NewPac(flag, id, content) + if err != nil { s.Close() - return err + logs.Warn("new pack err", err) + return } - return nil + err = pack.Pack(buf) + if err != nil { + s.Close() + logs.Warn("pack err", err) + return + } + _, err = buf.WriteTo(s.conn) + if err != nil { + s.Close() + logs.Warn("write err", err) + } + return } func (s *Mux) ping() { @@ -117,7 +116,7 @@ func (s *Mux) ping() { if (math.MaxInt32 - s.id) < 10000 { s.id = 0 } - if err := s.sendInfo(MUX_PING_FLAG, MUX_PING, nil); err != nil || (s.pingOk > 10 && s.connType == "kcp") { + if err := s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, nil); err != nil || (s.pingOk > 10 && s.connType == "kcp") { s.Close() break } @@ -130,65 +129,48 @@ func (s *Mux) ping() { } func (s *Mux) readSession() { - var buf []byte + var pack common.MuxPackager go func() { for { - var flag, i int32 - var n int - var err error - if binary.Read(s.conn, binary.LittleEndian, &flag) == nil { - if binary.Read(s.conn, binary.LittleEndian, &i) != nil { - break - } - s.pingOk = 0 - switch flag { - case MUX_NEW_CONN: //new conn - conn := NewConn(i, s) - s.connMap.Set(i, conn) //it has been set before send ok - s.newConnCh <- conn - s.sendInfo(MUX_NEW_CONN_OK, i, nil) - continue - case MUX_PING_FLAG: //ping - s.sendInfo(MUX_PING_RETURN, MUX_PING, nil) - continue - case MUX_PING_RETURN: - continue - case MUX_NEW_MSG: - buf = pool.GetBufPoolCopy() - if n, err = ReadLenBytes(buf, s.conn); err != nil { - break - } - } - if conn, ok := s.connMap.Get(i); ok && !conn.isClose { - switch flag { - case MUX_NEW_MSG: //new msg from remote conn - //insert wait queue - conn.waitQueue.Push(NewBufNode(buf, n)) - //judge len if >xxx ,send stop - if conn.readWait { - conn.readWait = false - conn.readCh <- struct{}{} - } - case MUX_MSG_SEND_OK: //the remote has read - select { - case conn.getStatusCh <- struct{}{}: - default: - } - conn.hasWrite -- - case MUX_NEW_CONN_OK: //conn ok - conn.connStatusOkCh <- struct{}{} - case MUX_NEW_CONN_Fail: - conn.connStatusFailCh <- struct{}{} - case MUX_CONN_CLOSE: //close the connection - go conn.Close() - s.connMap.Delete(i) - } - } else if flag == MUX_NEW_MSG { - pool.PutBufPoolCopy(buf) - } - } else { + if pack.UnPack(s.conn) != nil { break } + s.pingOk = 0 + switch pack.Flag { + case common.MUX_NEW_CONN: //new conn + logs.Warn("mux new conn", pack.Id) + conn := NewConn(pack.Id, s) + s.connMap.Set(pack.Id, conn) //it has been set before send ok + s.newConnCh <- conn + s.sendInfo(common.MUX_NEW_CONN_OK, pack.Id, nil) + continue + case common.MUX_PING_FLAG: //ping + s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, nil) + continue + case common.MUX_PING_RETURN: + continue + } + if conn, ok := s.connMap.Get(pack.Id); ok && !conn.isClose { + switch pack.Flag { + case common.MUX_NEW_MSG: //new msg from remote conn + //insert wait queue + conn.waitQueue.Push(NewBufNode(pack.Content, int(pack.Length))) + //judge len if >xxx ,send stop + if conn.readWait { + conn.readWait = false + conn.readCh <- struct{}{} + } + case common.MUX_NEW_CONN_OK: //conn ok + conn.connStatusOkCh <- struct{}{} + case common.MUX_NEW_CONN_Fail: + conn.connStatusFailCh <- struct{}{} + case common.MUX_CONN_CLOSE: //close the connection + conn.waitQueue.Push(NewBufNode(nil, 0)) + s.connMap.Delete(pack.Id) + } + } else if pack.Flag == common.MUX_NEW_MSG { + pool.PutBufPoolCopy(pack.Content) + } } s.Close() }() @@ -198,6 +180,7 @@ func (s *Mux) readSession() { } func (s *Mux) Close() error { + logs.Warn("close mux") if s.IsClose { return errors.New("the mux has closed") } @@ -214,6 +197,10 @@ func (s *Mux) Close() error { } //get new connId as unique flag -func (s *Mux) getId() int32 { - return atomic.AddInt32(&s.id, 1) +func (s *Mux) getId() (id int32) { + id = atomic.AddInt32(&s.id, 1) + if _, ok := s.connMap.Get(id); ok { + s.getId() + } + return } diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index f84e378..067e939 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -32,13 +32,14 @@ func TestNewMux(t *testing.T) { log.Fatalln(err) } go func(c net.Conn) { - c2, err := net.Dial("tcp", "127.0.0.1:8082") + c2, err := net.Dial("tcp", "127.0.0.1:80") if err != nil { log.Fatalln(err) } go common.CopyBuffer(c2, c) common.CopyBuffer(c, c2) c.Close() + //logs.Warn("close from out npc ") c2.Close() }(c) } @@ -64,6 +65,7 @@ func TestNewMux(t *testing.T) { common.CopyBuffer(conn, tmpCpnn) conn.Close() tmpCpnn.Close() + logs.Warn("close from out nps ", tmpCpnn.connId) }(conn) } }() diff --git a/lib/pool/pool.go b/lib/pool/pool.go index 70e0477..fb337a2 100644 --- a/lib/pool/pool.go +++ b/lib/pool/pool.go @@ -1,6 +1,7 @@ package pool import ( + "bytes" "sync" ) @@ -36,6 +37,7 @@ var BufPoolCopy = sync.Pool{ return &buf }, } + func PutBufPoolUdp(buf []byte) { if cap(buf) == PoolSizeUdp { BufPoolUdp.Put(buf[:PoolSizeUdp]) @@ -48,7 +50,7 @@ func PutBufPoolCopy(buf []byte) { } } -func GetBufPoolCopy() ([]byte) { +func GetBufPoolCopy() []byte { return (*BufPoolCopy.Get().(*[]byte))[:PoolSizeCopy] } @@ -57,3 +59,31 @@ func PutBufPoolMax(buf []byte) { BufPoolMax.Put(buf[:PoolSize]) } } + +type BufferPool struct { + pool sync.Pool +} + +func (Self *BufferPool) New() { + Self.pool = sync.Pool{ + New: func() interface{} { + return new(bytes.Buffer) + }, + } +} + +func (Self *BufferPool) Get() *bytes.Buffer { + return Self.pool.Get().(*bytes.Buffer) +} + +func (Self *BufferPool) Put(x *bytes.Buffer) { + x.Reset() + Self.pool.Put(x) +} + +var once = sync.Once{} +var BuffPool = BufferPool{} + +func init() { + once.Do(BuffPool.New) +} From f35a73f7347448c996a5bd7faa71024f901a4d9e Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Mon, 26 Aug 2019 18:47:23 +0800 Subject: [PATCH 02/38] mux test --- lib/common/netpackager.go | 2 +- lib/common/util.go | 3 +++ lib/mux/conn.go | 35 +++++++++++++++++++++------ lib/mux/mux.go | 51 ++++++++++++++++++++++++++++++++++----- lib/mux/mux_test.go | 20 +++++++++------ 5 files changed, 89 insertions(+), 22 deletions(-) diff --git a/lib/common/netpackager.go b/lib/common/netpackager.go index 315f645..8e91de8 100644 --- a/lib/common/netpackager.go +++ b/lib/common/netpackager.go @@ -45,13 +45,13 @@ func (Self *BasePackager) Pack(writer io.Writer) (err error) { return } err = binary.Write(writer, binary.LittleEndian, Self.Content) - //logs.Warn(Self.Length, string(Self.Content)) return } //Unpack 会导致传入的数字类型转化成float64！！ //主要原因是json unmarshal并未传入正确的数据类型 func (Self *BasePackager) UnPack(reader io.Reader) (err error) { + Self.clean() err = binary.Read(reader, binary.LittleEndian, &Self.Length) if err != nil { return diff --git a/lib/common/util.go b/lib/common/util.go index ce4381d..ce2b896 100755 --- a/lib/common/util.go +++ b/lib/common/util.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "github.com/cnlh/nps/lib/crypt" "github.com/cnlh/nps/lib/pool" + "github.com/cnlh/nps/vender/github.com/astaxie/beego/logs" "html/template" "io" "io/ioutil" @@ -268,8 +269,10 @@ func CopyBuffer(dst io.Writer, src io.Reader) (written int64, err error) { defer pool.PutBufPoolCopy(buf) for { nr, er := src.Read(buf) + logs.Warn("read finish", nr, er) if nr > 0 { nw, ew := dst.Write(buf[0:nr]) + logs.Warn("write finish", nw, ew) if nw > 0 { written += int64(nw) } diff --git a/lib/mux/conn.go b/lib/mux/conn.go index a14e98d..cbaf935 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -22,11 +22,12 @@ type conn struct { endRead int //now end read readFlag bool readCh chan struct{} - waitQueue *sliceEntry + readQueue *sliceEntry stopWrite bool connId int32 isClose bool readWait bool + sendClose bool hasWrite int mux *Mux } @@ -37,7 +38,7 @@ func NewConn(connId int32, mux *Mux) *conn { getStatusCh: make(chan struct{}), connStatusOkCh: make(chan struct{}), connStatusFailCh: make(chan struct{}), - waitQueue: NewQueue(), + readQueue: NewQueue(), connId: connId, mux: mux, } @@ -45,11 +46,12 @@ func NewConn(connId int32, mux *Mux) *conn { } func (s *conn) Read(buf []byte) (n int, err error) { + logs.Warn("starting read ", s.connId) if s.isClose || buf == nil { return 0, errors.New("the conn has closed") } if s.endRead-s.startRead == 0 { //read finish or start - if s.waitQueue.Size() == 0 { + if s.readQueue.Size() == 0 { s.readWait = true if t := s.readTimeOut.Sub(time.Now()); t > 0 { timer := time.NewTimer(t) @@ -67,19 +69,22 @@ func (s *conn) Read(buf []byte) (n int, err error) { if s.isClose { //If the connection is closed instead of continuing command return 0, errors.New("the conn has closed") } - if node, err := s.waitQueue.Pop(); err != nil { + if node, err := s.readQueue.Pop(); err != nil { s.Close() return 0, io.EOF } else { pool.PutBufPoolCopy(s.readBuffer) if node.val == nil { //close + s.sendClose = true s.Close() logs.Warn("close from read msg ", s.connId) + return 0, io.EOF } else { s.readBuffer = node.val s.endRead = node.l s.startRead = 0 + logs.Warn("get a new data buffer ", s.connId) } } } @@ -90,10 +95,12 @@ func (s *conn) Read(buf []byte) (n int, err error) { n = copy(buf, s.readBuffer[s.startRead:s.endRead]) s.startRead += n } + logs.Warn("end read ", s.connId) return } func (s *conn) Write(buf []byte) (n int, err error) { + logs.Warn("trying write", s.connId) if s.isClose { return 0, errors.New("the conn has closed") } @@ -113,6 +120,7 @@ func (s *conn) Write(buf []byte) (n int, err error) { if s.isClose { return 0, io.EOF } + logs.Warn("write success ", s.connId) return len(buf), nil } func (s *conn) write(buf []byte, ch chan struct{}) { @@ -130,7 +138,8 @@ func (s *conn) write(buf []byte, ch chan struct{}) { ch <- struct{}{} } -func (s *conn) Close() error { +func (s *conn) Close() (err error) { + logs.Warn("start closing ", s.connId) if s.isClose { logs.Warn("already closed", s.connId) return errors.New("the conn has closed") @@ -140,12 +149,22 @@ func (s *conn) Close() error { if s.readWait { s.readCh <- struct{}{} } - s.waitQueue.Clear() + s.readQueue.Clear() s.mux.connMap.Delete(s.connId) if !s.mux.IsClose { - s.mux.sendInfo(common.MUX_CONN_CLOSE, s.connId, nil) + if !s.sendClose { + logs.Warn("start send closing msg", s.connId) + err = s.mux.sendInfo(common.MUX_CONN_CLOSE, s.connId, nil) + logs.Warn("send closing msg ok ", s.connId) + if err != nil { + logs.Warn(err) + return + } + } else { + logs.Warn("send mux conn close pass ", s.connId) + } } - return nil + return } func (s *conn) LocalAddr() net.Addr { diff --git a/lib/mux/mux.go b/lib/mux/mux.go index bfd82ff..c099d51 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -1,6 +1,7 @@ package mux import ( + "bytes" "errors" "github.com/cnlh/nps/lib/common" "github.com/cnlh/nps/lib/pool" @@ -22,7 +23,7 @@ type Mux struct { IsClose bool pingOk int connType string - writeQueue *sliceEntry + writeQueue chan *bytes.Buffer sync.Mutex } @@ -35,12 +36,13 @@ func NewMux(c net.Conn, connType string) *Mux { newConnCh: make(chan *conn), IsClose: false, connType: connType, - writeQueue: NewQueue(), + writeQueue: make(chan *bytes.Buffer, 20), } //read session by flag go m.readSession() //ping go m.ping() + //go m.writeSession() return m } @@ -82,8 +84,10 @@ func (s *Mux) Addr() net.Addr { } func (s *Mux) sendInfo(flag uint8, id int32, content []byte) (err error) { + if flag == common.MUX_NEW_MSG { + logs.Warn("trying write to mux new msg", id) + } buf := pool.BuffPool.Get() - defer pool.BuffPool.Put(buf) pack := common.MuxPackager{} err = pack.NewPac(flag, id, content) if err != nil { @@ -97,14 +101,39 @@ func (s *Mux) sendInfo(flag uint8, id int32, content []byte) (err error) { logs.Warn("pack err", err) return } + //s.writeQueue <- buf _, err = buf.WriteTo(s.conn) if err != nil { s.Close() logs.Warn("write err", err) } + pool.BuffPool.Put(buf) + if flag == common.MUX_CONN_CLOSE { + logs.Warn("write to mux conn close success", id) + } + if flag == common.MUX_NEW_MSG { + logs.Warn("write to mux new msg success", id) + } return } +func (s *Mux) writeSession() { + go func() { + for { + buf := <-s.writeQueue + l := buf.Len() + n, err := buf.WriteTo(s.conn) + pool.BuffPool.Put(buf) + if err != nil || int(n) != l { + logs.Warn("close from write to ", err, n, l) + s.Close() + break + } + } + }() + <-s.closeChan +} + func (s *Mux) ping() { go func() { ticker := time.NewTicker(time.Second * 1) @@ -138,7 +167,7 @@ func (s *Mux) readSession() { s.pingOk = 0 switch pack.Flag { case common.MUX_NEW_CONN: //new conn - logs.Warn("mux new conn", pack.Id) + //logs.Warn("mux new conn", pack.Id) conn := NewConn(pack.Id, s) s.connMap.Set(pack.Id, conn) //it has been set before send ok s.newConnCh <- conn @@ -151,22 +180,30 @@ func (s *Mux) readSession() { continue } if conn, ok := s.connMap.Get(pack.Id); ok && !conn.isClose { + logs.Warn("read session flag id", pack.Flag, pack.Id) switch pack.Flag { case common.MUX_NEW_MSG: //new msg from remote conn //insert wait queue - conn.waitQueue.Push(NewBufNode(pack.Content, int(pack.Length))) + conn.readQueue.Push(NewBufNode(pack.Content, int(pack.Length))) //judge len if >xxx ,send stop if conn.readWait { conn.readWait = false conn.readCh <- struct{}{} } + logs.Warn("push a read buffer ", conn.connId, pack.Id) case common.MUX_NEW_CONN_OK: //conn ok conn.connStatusOkCh <- struct{}{} case common.MUX_NEW_CONN_Fail: conn.connStatusFailCh <- struct{}{} case common.MUX_CONN_CLOSE: //close the connection - conn.waitQueue.Push(NewBufNode(nil, 0)) + conn.readQueue.Push(NewBufNode(nil, 0)) + if conn.readWait { + logs.Warn("close read wait", pack.Id) + conn.readWait = false + conn.readCh <- struct{}{} + } s.connMap.Delete(pack.Id) + logs.Warn("read session mux conn close finish", pack.Id) } } else if pack.Flag == common.MUX_NEW_MSG { pool.PutBufPoolCopy(pack.Content) @@ -192,6 +229,8 @@ func (s *Mux) Close() error { select { case s.closeChan <- struct{}{}: } + s.closeChan <- struct{}{} + close(s.writeQueue) close(s.newConnCh) return s.conn.Close() } diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index 067e939..5b571d0 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -26,22 +26,25 @@ func TestNewMux(t *testing.T) { time.Sleep(time.Second * 3) go func() { m2 := NewMux(conn2, "tcp") + connCh := make(chan bool, 1) for { c, err := m2.Accept() if err != nil { log.Fatalln(err) } - go func(c net.Conn) { + connCh <- true + go func(c net.Conn, ch chan bool) { c2, err := net.Dial("tcp", "127.0.0.1:80") if err != nil { log.Fatalln(err) } go common.CopyBuffer(c2, c) common.CopyBuffer(c, c2) - c.Close() - //logs.Warn("close from out npc ") c2.Close() - }(c) + c.Close() + logs.Warn("close npc") + <-ch + }(c, connCh) } }() @@ -51,12 +54,14 @@ func TestNewMux(t *testing.T) { if err != nil { log.Fatalln(err) } + connCh := make(chan bool, 1) for { conn, err := l.Accept() if err != nil { log.Fatalln(err) } - go func(conn net.Conn) { + connCh <- true + go func(conn net.Conn, ch chan bool) { tmpCpnn, err := m1.NewConn() if err != nil { log.Fatalln(err) @@ -64,9 +69,10 @@ func TestNewMux(t *testing.T) { go common.CopyBuffer(tmpCpnn, conn) common.CopyBuffer(conn, tmpCpnn) conn.Close() - tmpCpnn.Close() + //tmpCpnn.Close() logs.Warn("close from out nps ", tmpCpnn.connId) - }(conn) + <-ch + }(conn, connCh) } }() From 4bb7e33b16688539e230e77467fa8dc99ee35e7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=B2=B3?= Date: Mon, 26 Aug 2019 21:42:20 +0800 Subject: [PATCH 03/38] no message --- lib/common/netpackager.go | 1 + lib/common/util.go | 5 ++-- lib/mux/conn.go | 12 +--------- lib/mux/mux.go | 11 ++++----- lib/mux/mux_test.go | 48 ++++++++++++++++----------------------- lib/pool/pool.go | 7 +++--- 6 files changed, 32 insertions(+), 52 deletions(-) diff --git a/lib/common/netpackager.go b/lib/common/netpackager.go index 8e91de8..1252710 100644 --- a/lib/common/netpackager.go +++ b/lib/common/netpackager.go @@ -160,6 +160,7 @@ func (Self *MuxPackager) Pack(writer io.Writer) (err error) { } func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { + Self.Length=0 err = binary.Read(reader, binary.LittleEndian, &Self.Flag) if err != nil { return diff --git a/lib/common/util.go b/lib/common/util.go index ce2b896..812c543 100755 --- a/lib/common/util.go +++ b/lib/common/util.go @@ -264,15 +264,14 @@ func GetPortByAddr(addr string) int { return p } -func CopyBuffer(dst io.Writer, src io.Reader) (written int64, err error) { +func CopyBuffer(dst io.Writer, src io.Reader,connId int32) (written int64, err error) { buf := pool.GetBufPoolCopy() defer pool.PutBufPoolCopy(buf) for { nr, er := src.Read(buf) - logs.Warn("read finish", nr, er) if nr > 0 { + logs.Warn("write",connId, nr, string(buf[0:10])) nw, ew := dst.Write(buf[0:nr]) - logs.Warn("write finish", nw, ew) if nw > 0 { written += int64(nw) } diff --git a/lib/mux/conn.go b/lib/mux/conn.go index cbaf935..6c6ab95 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -46,7 +46,6 @@ func NewConn(connId int32, mux *Mux) *conn { } func (s *conn) Read(buf []byte) (n int, err error) { - logs.Warn("starting read ", s.connId) if s.isClose || buf == nil { return 0, errors.New("the conn has closed") } @@ -73,18 +72,16 @@ func (s *conn) Read(buf []byte) (n int, err error) { s.Close() return 0, io.EOF } else { - pool.PutBufPoolCopy(s.readBuffer) + //pool.PutBufPoolCopy(s.readBuffer) if node.val == nil { //close s.sendClose = true s.Close() - logs.Warn("close from read msg ", s.connId) return 0, io.EOF } else { s.readBuffer = node.val s.endRead = node.l s.startRead = 0 - logs.Warn("get a new data buffer ", s.connId) } } } @@ -95,12 +92,10 @@ func (s *conn) Read(buf []byte) (n int, err error) { n = copy(buf, s.readBuffer[s.startRead:s.endRead]) s.startRead += n } - logs.Warn("end read ", s.connId) return } func (s *conn) Write(buf []byte) (n int, err error) { - logs.Warn("trying write", s.connId) if s.isClose { return 0, errors.New("the conn has closed") } @@ -120,7 +115,6 @@ func (s *conn) Write(buf []byte) (n int, err error) { if s.isClose { return 0, io.EOF } - logs.Warn("write success ", s.connId) return len(buf), nil } func (s *conn) write(buf []byte, ch chan struct{}) { @@ -139,9 +133,7 @@ func (s *conn) write(buf []byte, ch chan struct{}) { } func (s *conn) Close() (err error) { - logs.Warn("start closing ", s.connId) if s.isClose { - logs.Warn("already closed", s.connId) return errors.New("the conn has closed") } s.isClose = true @@ -153,7 +145,6 @@ func (s *conn) Close() (err error) { s.mux.connMap.Delete(s.connId) if !s.mux.IsClose { if !s.sendClose { - logs.Warn("start send closing msg", s.connId) err = s.mux.sendInfo(common.MUX_CONN_CLOSE, s.connId, nil) logs.Warn("send closing msg ok ", s.connId) if err != nil { @@ -161,7 +152,6 @@ func (s *conn) Close() (err error) { return } } else { - logs.Warn("send mux conn close pass ", s.connId) } } return diff --git a/lib/mux/mux.go b/lib/mux/mux.go index c099d51..e6daae7 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -85,7 +85,6 @@ func (s *Mux) Addr() net.Addr { func (s *Mux) sendInfo(flag uint8, id int32, content []byte) (err error) { if flag == common.MUX_NEW_MSG { - logs.Warn("trying write to mux new msg", id) } buf := pool.BuffPool.Get() pack := common.MuxPackager{} @@ -109,10 +108,8 @@ func (s *Mux) sendInfo(flag uint8, id int32, content []byte) (err error) { } pool.BuffPool.Put(buf) if flag == common.MUX_CONN_CLOSE { - logs.Warn("write to mux conn close success", id) } if flag == common.MUX_NEW_MSG { - logs.Warn("write to mux new msg success", id) } return } @@ -164,6 +161,11 @@ func (s *Mux) readSession() { if pack.UnPack(s.conn) != nil { break } + if pack.Flag != 0 && pack.Flag != 7 { + if pack.Length>10 { + logs.Warn(pack.Flag, pack.Id, pack.Length,string(pack.Content[:10])) + } + } s.pingOk = 0 switch pack.Flag { case common.MUX_NEW_CONN: //new conn @@ -180,7 +182,6 @@ func (s *Mux) readSession() { continue } if conn, ok := s.connMap.Get(pack.Id); ok && !conn.isClose { - logs.Warn("read session flag id", pack.Flag, pack.Id) switch pack.Flag { case common.MUX_NEW_MSG: //new msg from remote conn //insert wait queue @@ -190,7 +191,6 @@ func (s *Mux) readSession() { conn.readWait = false conn.readCh <- struct{}{} } - logs.Warn("push a read buffer ", conn.connId, pack.Id) case common.MUX_NEW_CONN_OK: //conn ok conn.connStatusOkCh <- struct{}{} case common.MUX_NEW_CONN_Fail: @@ -203,7 +203,6 @@ func (s *Mux) readSession() { conn.readCh <- struct{}{} } s.connMap.Delete(pack.Id) - logs.Warn("read session mux conn close finish", pack.Id) } } else if pack.Flag == common.MUX_NEW_MSG { pool.PutBufPoolCopy(pack.Content) diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index 5b571d0..2d3d2d0 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -26,25 +26,20 @@ func TestNewMux(t *testing.T) { time.Sleep(time.Second * 3) go func() { m2 := NewMux(conn2, "tcp") - connCh := make(chan bool, 1) for { c, err := m2.Accept() if err != nil { log.Fatalln(err) } - connCh <- true - go func(c net.Conn, ch chan bool) { - c2, err := net.Dial("tcp", "127.0.0.1:80") - if err != nil { - log.Fatalln(err) - } - go common.CopyBuffer(c2, c) - common.CopyBuffer(c, c2) - c2.Close() - c.Close() - logs.Warn("close npc") - <-ch - }(c, connCh) + c2, err := net.Dial("tcp", "127.0.0.1:8080") + if err != nil { + log.Fatalln(err) + } + go common.CopyBuffer(c2, c,0) + common.CopyBuffer(c, c2,0) + c2.Close() + c.Close() + logs.Warn("close npc") } }() @@ -54,25 +49,22 @@ func TestNewMux(t *testing.T) { if err != nil { log.Fatalln(err) } - connCh := make(chan bool, 1) for { conn, err := l.Accept() if err != nil { log.Fatalln(err) } - connCh <- true - go func(conn net.Conn, ch chan bool) { - tmpCpnn, err := m1.NewConn() - if err != nil { - log.Fatalln(err) - } - go common.CopyBuffer(tmpCpnn, conn) - common.CopyBuffer(conn, tmpCpnn) - conn.Close() - //tmpCpnn.Close() - logs.Warn("close from out nps ", tmpCpnn.connId) - <-ch - }(conn, connCh) + + tmpCpnn, err := m1.NewConn() + if err != nil { + log.Fatalln(err) + } + go common.CopyBuffer(tmpCpnn, conn,tmpCpnn.connId) + _, err = common.CopyBuffer(conn, tmpCpnn,tmpCpnn.connId) + logs.Warn(err, tmpCpnn.connId) + conn.Close() + tmpCpnn.Close() + logs.Warn("close from out nps ", tmpCpnn.connId) } }() diff --git a/lib/pool/pool.go b/lib/pool/pool.go index fb337a2..e491a5e 100644 --- a/lib/pool/pool.go +++ b/lib/pool/pool.go @@ -33,8 +33,7 @@ var BufPoolSmall = sync.Pool{ } var BufPoolCopy = sync.Pool{ New: func() interface{} { - buf := make([]byte, PoolSizeCopy) - return &buf + return make([]byte, PoolSizeCopy) }, } @@ -46,12 +45,12 @@ func PutBufPoolUdp(buf []byte) { func PutBufPoolCopy(buf []byte) { if cap(buf) == PoolSizeCopy { - BufPoolCopy.Put(&buf) + BufPoolCopy.Put(buf[:PoolSizeCopy]) } } func GetBufPoolCopy() []byte { - return (*BufPoolCopy.Get().(*[]byte))[:PoolSizeCopy] + return (BufPoolCopy.Get().([]byte))[:PoolSizeCopy] } func PutBufPoolMax(buf []byte) { From 53c2e472ae5dd79b86c3584ca11e13dfeb3ab1de Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Mon, 26 Aug 2019 23:29:22 +0800 Subject: [PATCH 04/38] Change BuffSizeCopy pool --- lib/common/netpackager.go | 8 +++++--- lib/mux/conn.go | 4 ++-- lib/mux/mux.go | 6 +++--- lib/mux/queue.go | 2 +- lib/pool/pool.go | 29 ++++++++++++++++++++++++++++- 5 files changed, 39 insertions(+), 10 deletions(-) diff --git a/lib/common/netpackager.go b/lib/common/netpackager.go index 1252710..abaff31 100644 --- a/lib/common/netpackager.go +++ b/lib/common/netpackager.go @@ -20,6 +20,7 @@ type BasePackager struct { } func (Self *BasePackager) NewPac(contents ...interface{}) (err error) { + Self.Content = pool.CopyBuff.Get() Self.clean() for _, content := range contents { switch content.(type) { @@ -45,6 +46,7 @@ func (Self *BasePackager) Pack(writer io.Writer) (err error) { return } err = binary.Write(writer, binary.LittleEndian, Self.Content) + pool.CopyBuff.Put(Self.Content) return } @@ -56,13 +58,13 @@ func (Self *BasePackager) UnPack(reader io.Reader) (err error) { if err != nil { return } - Self.Content = pool.GetBufPoolCopy() + Self.Content = pool.CopyBuff.Get() Self.Content = Self.Content[:Self.Length] //n, err := io.ReadFull(reader, Self.Content) //if n != int(Self.Length) { // err = io.ErrUnexpectedEOF //} - err = binary.Read(reader, binary.LittleEndian, &Self.Content) + err = binary.Read(reader, binary.LittleEndian, Self.Content) return } @@ -160,7 +162,7 @@ func (Self *MuxPackager) Pack(writer io.Writer) (err error) { } func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { - Self.Length=0 + Self.Length = 0 err = binary.Read(reader, binary.LittleEndian, &Self.Flag) if err != nil { return diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 6c6ab95..d30bedc 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -72,7 +72,6 @@ func (s *conn) Read(buf []byte) (n int, err error) { s.Close() return 0, io.EOF } else { - //pool.PutBufPoolCopy(s.readBuffer) if node.val == nil { //close s.sendClose = true @@ -91,6 +90,7 @@ func (s *conn) Read(buf []byte) (n int, err error) { } else { n = copy(buf, s.readBuffer[s.startRead:s.endRead]) s.startRead += n + pool.CopyBuff.Put(s.readBuffer) } return } @@ -137,7 +137,7 @@ func (s *conn) Close() (err error) { return errors.New("the conn has closed") } s.isClose = true - pool.PutBufPoolCopy(s.readBuffer) + pool.CopyBuff.Put(s.readBuffer) if s.readWait { s.readCh <- struct{}{} } diff --git a/lib/mux/mux.go b/lib/mux/mux.go index e6daae7..2073593 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -162,8 +162,8 @@ func (s *Mux) readSession() { break } if pack.Flag != 0 && pack.Flag != 7 { - if pack.Length>10 { - logs.Warn(pack.Flag, pack.Id, pack.Length,string(pack.Content[:10])) + if pack.Length > 10 { + logs.Warn(pack.Flag, pack.Id, pack.Length, string(pack.Content[:10])) } } s.pingOk = 0 @@ -205,7 +205,7 @@ func (s *Mux) readSession() { s.connMap.Delete(pack.Id) } } else if pack.Flag == common.MUX_NEW_MSG { - pool.PutBufPoolCopy(pack.Content) + pool.CopyBuff.Put(pack.Content) } } s.Close() diff --git a/lib/mux/queue.go b/lib/mux/queue.go index f03bafd..6a14a8d 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -63,7 +63,7 @@ func (entry *sliceEntry) Clear() bool { return false } for i := 0; i < entry.Size(); i++ { - pool.PutBufPoolCopy(entry.element[i].val) + pool.CopyBuff.Put(entry.element[i].val) entry.element[i] = nil } entry.element = nil diff --git a/lib/pool/pool.go b/lib/pool/pool.go index e491a5e..26a91f5 100644 --- a/lib/pool/pool.go +++ b/lib/pool/pool.go @@ -59,6 +59,27 @@ func PutBufPoolMax(buf []byte) { } } +type CopyBufferPool struct { + pool sync.Pool +} + +func (Self *CopyBufferPool) New() { + Self.pool = sync.Pool{ + New: func() interface{} { + return make([]byte, PoolSizeCopy) + }, + } +} + +func (Self *CopyBufferPool) Get() []byte { + return Self.pool.Get().([]byte) +} + +func (Self *CopyBufferPool) Put(x []byte) { + x = x[:0] + Self.pool.Put(x) +} + type BufferPool struct { pool sync.Pool } @@ -82,7 +103,13 @@ func (Self *BufferPool) Put(x *bytes.Buffer) { var once = sync.Once{} var BuffPool = BufferPool{} +var CopyBuff = CopyBufferPool{} + +func newPool() { + BuffPool.New() + CopyBuff.New() +} func init() { - once.Do(BuffPool.New) + once.Do(newPool) } From 41c282b38bef8553c7d12c9eef321887042159e9 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Tue, 27 Aug 2019 20:07:37 +0800 Subject: [PATCH 05/38] fix buffer size bug --- lib/common/netpackager.go | 28 +++++++++++++++++++++------- lib/common/util.go | 20 ++++++++++---------- lib/mux/mux.go | 2 +- lib/pool/pool.go | 5 +++-- 4 files changed, 35 insertions(+), 20 deletions(-) diff --git a/lib/common/netpackager.go b/lib/common/netpackager.go index abaff31..2d589aa 100644 --- a/lib/common/netpackager.go +++ b/lib/common/netpackager.go @@ -27,10 +27,13 @@ func (Self *BasePackager) NewPac(contents ...interface{}) (err error) { case nil: Self.Content = Self.Content[:0] case []byte: - Self.Content = append(Self.Content, content.([]byte)...) + err = Self.appendByte(content.([]byte)) case string: - Self.Content = append(Self.Content, []byte(content.(string))...) - Self.Content = append(Self.Content, []byte(CONN_DATA_SEQ)...) + err = Self.appendByte([]byte(content.(string))) + if err != nil { + return + } + err = Self.appendByte([]byte(CONN_DATA_SEQ)) default: err = Self.marshal(content) } @@ -39,6 +42,18 @@ func (Self *BasePackager) NewPac(contents ...interface{}) (err error) { return } +func (Self *BasePackager) appendByte(data []byte) (err error) { + m := len(Self.Content) + n := m + len(data) + if n <= cap(Self.Content) { + Self.Content = Self.Content[0:n] // grow the length for copy + copy(Self.Content[m:n], data) + return nil + } else { + return bytes.ErrTooLarge + } +} + //似乎这里涉及到父类作用域问题，当子类调用父类的方法时，其struct仅仅为父类的 func (Self *BasePackager) Pack(writer io.Writer) (err error) { err = binary.Write(writer, binary.LittleEndian, Self.Length) @@ -53,12 +68,12 @@ func (Self *BasePackager) Pack(writer io.Writer) (err error) { //Unpack 会导致传入的数字类型转化成float64！！ //主要原因是json unmarshal并未传入正确的数据类型 func (Self *BasePackager) UnPack(reader io.Reader) (err error) { + Self.Content = pool.CopyBuff.Get() Self.clean() err = binary.Read(reader, binary.LittleEndian, &Self.Length) if err != nil { return } - Self.Content = pool.CopyBuff.Get() Self.Content = Self.Content[:Self.Length] //n, err := io.ReadFull(reader, Self.Content) //if n != int(Self.Length) { @@ -73,7 +88,7 @@ func (Self *BasePackager) marshal(content interface{}) (err error) { if err != nil { return err } - Self.Content = append(Self.Content, tmp...) + err = Self.appendByte(tmp) return } @@ -92,7 +107,7 @@ func (Self *BasePackager) setLength() { func (Self *BasePackager) clean() { Self.Length = 0 - Self.Content = Self.Content[:0] + Self.Content = Self.Content[:0] // reset length } func (Self *BasePackager) Split() (strList []string) { @@ -162,7 +177,6 @@ func (Self *MuxPackager) Pack(writer io.Writer) (err error) { } func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { - Self.Length = 0 err = binary.Read(reader, binary.LittleEndian, &Self.Flag) if err != nil { return diff --git a/lib/common/util.go b/lib/common/util.go index 812c543..25cfee2 100755 --- a/lib/common/util.go +++ b/lib/common/util.go @@ -264,13 +264,19 @@ func GetPortByAddr(addr string) int { return p } -func CopyBuffer(dst io.Writer, src io.Reader,connId int32) (written int64, err error) { - buf := pool.GetBufPoolCopy() - defer pool.PutBufPoolCopy(buf) +func CopyBuffer(dst io.Writer, src io.Reader, connId int32) (written int64, err error) { + buf := pool.CopyBuff.Get() + defer pool.CopyBuff.Put(buf) for { nr, er := src.Read(buf) + if er != nil { + if er != io.EOF { + err = er + } + break + } if nr > 0 { - logs.Warn("write",connId, nr, string(buf[0:10])) + logs.Warn("write", connId, nr, string(buf[0:10])) nw, ew := dst.Write(buf[0:nr]) if nw > 0 { written += int64(nw) @@ -284,12 +290,6 @@ func CopyBuffer(dst io.Writer, src io.Reader,connId int32) (written int64, err e break } } - if er != nil { - if er != io.EOF { - err = er - } - break - } } return written, err } diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 2073593..ad17cb0 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -228,7 +228,7 @@ func (s *Mux) Close() error { select { case s.closeChan <- struct{}{}: } - s.closeChan <- struct{}{} + //s.closeChan <- struct{}{} close(s.writeQueue) close(s.newConnCh) return s.conn.Close() diff --git a/lib/pool/pool.go b/lib/pool/pool.go index 26a91f5..0540a9d 100644 --- a/lib/pool/pool.go +++ b/lib/pool/pool.go @@ -66,13 +66,14 @@ type CopyBufferPool struct { func (Self *CopyBufferPool) New() { Self.pool = sync.Pool{ New: func() interface{} { - return make([]byte, PoolSizeCopy) + return make([]byte, PoolSizeCopy, PoolSizeCopy) }, } } func (Self *CopyBufferPool) Get() []byte { - return Self.pool.Get().([]byte) + buf := Self.pool.Get().([]byte) + return buf[:cap(buf)] // grow to capacity } func (Self *CopyBufferPool) Put(x []byte) { From bb2cffe10a1f296337f3b72643dea5631ddc2789 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Fri, 30 Aug 2019 20:05:09 +0800 Subject: [PATCH 06/38] Change pool, change mux connection close ,change copy buffer --- lib/common/netpackager.go | 16 +++-- lib/{pool => common}/pool.go | 38 ++++++++++-- lib/common/util.go | 114 +++++++++++++++++++++++++++-------- lib/conn/conn.go | 11 ++-- lib/conn/snappy.go | 6 +- lib/mux/conn.go | 42 +++++++------ lib/mux/mux.go | 74 ++++++++++++++--------- lib/mux/mux_test.go | 65 ++++++++++++-------- lib/mux/queue.go | 4 +- server/proxy/p2p.go | 3 +- server/proxy/udp.go | 7 +-- 11 files changed, 257 insertions(+), 123 deletions(-) rename lib/{pool => common}/pool.go (71%) diff --git a/lib/common/netpackager.go b/lib/common/netpackager.go index 2d589aa..6d67f44 100644 --- a/lib/common/netpackager.go +++ b/lib/common/netpackager.go @@ -4,7 +4,8 @@ import ( "bytes" "encoding/binary" "encoding/json" - "github.com/cnlh/nps/lib/pool" + "errors" + "github.com/cnlh/nps/vender/github.com/astaxie/beego/logs" "io" "strings" ) @@ -20,7 +21,6 @@ type BasePackager struct { } func (Self *BasePackager) NewPac(contents ...interface{}) (err error) { - Self.Content = pool.CopyBuff.Get() Self.clean() for _, content := range contents { switch content.(type) { @@ -50,7 +50,8 @@ func (Self *BasePackager) appendByte(data []byte) (err error) { copy(Self.Content[m:n], data) return nil } else { - return bytes.ErrTooLarge + logs.Warn(len(data), len(Self.Content), cap(Self.Content)) + return errors.New("pack content too large") } } @@ -61,20 +62,22 @@ func (Self *BasePackager) Pack(writer io.Writer) (err error) { return } err = binary.Write(writer, binary.LittleEndian, Self.Content) - pool.CopyBuff.Put(Self.Content) return } //Unpack 会导致传入的数字类型转化成float64！！ //主要原因是json unmarshal并未传入正确的数据类型 func (Self *BasePackager) UnPack(reader io.Reader) (err error) { - Self.Content = pool.CopyBuff.Get() Self.clean() err = binary.Read(reader, binary.LittleEndian, &Self.Length) if err != nil { return } - Self.Content = Self.Content[:Self.Length] + if int(Self.Length) > cap(Self.Content) { + logs.Warn("unpack", cap(Self.Content)) + err = errors.New("unpack err, content length too large") + } + Self.Content = Self.Content[:int(Self.Length)] //n, err := io.ReadFull(reader, Self.Content) //if n != int(Self.Length) { // err = io.ErrUnexpectedEOF @@ -177,6 +180,7 @@ func (Self *MuxPackager) Pack(writer io.Writer) (err error) { } func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { + Self.BasePackager.clean() // also clean the content err = binary.Read(reader, binary.LittleEndian, &Self.Flag) if err != nil { return diff --git a/lib/pool/pool.go b/lib/common/pool.go similarity index 71% rename from lib/pool/pool.go rename to lib/common/pool.go index 0540a9d..205378d 100644 --- a/lib/pool/pool.go +++ b/lib/common/pool.go @@ -1,4 +1,4 @@ -package pool +package common import ( "bytes" @@ -73,12 +73,15 @@ func (Self *CopyBufferPool) New() { func (Self *CopyBufferPool) Get() []byte { buf := Self.pool.Get().([]byte) - return buf[:cap(buf)] // grow to capacity + return buf[:PoolSizeCopy] // just like make a new slice, but data may not be 0 } func (Self *CopyBufferPool) Put(x []byte) { - x = x[:0] - Self.pool.Put(x) + if len(x) == PoolSizeCopy { + Self.pool.Put(x) + } else { + x = nil // buf is not full, maybe truncated by gc in pool, not allowed + } } type BufferPool struct { @@ -102,13 +105,40 @@ func (Self *BufferPool) Put(x *bytes.Buffer) { Self.pool.Put(x) } +type MuxPackagerPool struct { + pool sync.Pool +} + +func (Self *MuxPackagerPool) New() { + Self.pool = sync.Pool{ + New: func() interface{} { + pack := MuxPackager{} + return &pack + }, + } +} + +func (Self *MuxPackagerPool) Get() *MuxPackager { + pack := Self.pool.Get().(*MuxPackager) + buf := CopyBuff.Get() + pack.Content = buf + return pack +} + +func (Self *MuxPackagerPool) Put(pack *MuxPackager) { + CopyBuff.Put(pack.Content) + Self.pool.Put(pack) +} + var once = sync.Once{} var BuffPool = BufferPool{} var CopyBuff = CopyBufferPool{} +var MuxPack = MuxPackagerPool{} func newPool() { BuffPool.New() CopyBuff.New() + MuxPack.New() } func init() { diff --git a/lib/common/util.go b/lib/common/util.go index 25cfee2..3b4369f 100755 --- a/lib/common/util.go +++ b/lib/common/util.go @@ -4,8 +4,8 @@ import ( "bytes" "encoding/base64" "encoding/binary" + "errors" "github.com/cnlh/nps/lib/crypt" - "github.com/cnlh/nps/lib/pool" "github.com/cnlh/nps/vender/github.com/astaxie/beego/logs" "html/template" "io" @@ -264,34 +264,96 @@ func GetPortByAddr(addr string) int { return p } -func CopyBuffer(dst io.Writer, src io.Reader, connId int32) (written int64, err error) { - buf := pool.CopyBuff.Get() - defer pool.CopyBuff.Put(buf) - for { - nr, er := src.Read(buf) - if er != nil { - if er != io.EOF { - err = er - } - break +type ConnCopy struct { + dst net.Conn + src net.Conn + buf []byte + connId int32 +} + +func (Self *ConnCopy) New(dst net.Conn, src net.Conn, connId int32) { + Self.dst = dst + Self.src = src + Self.buf = CopyBuff.Get() + Self.connId = connId +} + +func (Self *ConnCopy) copyBufferOnce() (written int64, err error) { + nr, er := Self.src.Read(Self.buf) + if nr > 0 { + //logs.Warn("write", Self.connId, nr, string(buf[0:10])) + nw, ew := Self.dst.Write(Self.buf[0:nr]) + if nw > 0 { + written = int64(nw) } - if nr > 0 { - logs.Warn("write", connId, nr, string(buf[0:10])) - nw, ew := dst.Write(buf[0:nr]) - if nw > 0 { - written += int64(nw) - } - if ew != nil { - err = ew - break - } - if nr != nw { - err = io.ErrShortWrite - break - } + if ew != nil { + //logs.Warn("write err ew id nw", ew, Self.connId, nw) + err = ew + return + } + if nr != nw { + err = io.ErrShortWrite + return + } + if nw == 0 { + err = errors.New("io: write on closed pipe") + //logs.Warn("write buffer", err) + return } } - return written, err + if nr == 0 && er == nil { + err = errors.New("io: read on closed pipe") + //logs.Warn("read buffer", err) + return + } + if er != nil { + err = er + return + } + return +} + +func (Self *ConnCopy) copyBuffer() (written int64, err error) { + var write int64 + write, err = Self.copyBufferOnce() // first copy, if written is zero and err is io.EOF + // means conn already closed, so need to close all the conn + written += write + if err == io.EOF && written == 0 { + err = errors.New("io: read on closed pipe") + return + } else if err == io.EOF && written > 0 { + err = nil + return + } + for { + write, err = Self.copyBufferOnce() + written += write + if err != nil { + if err == io.EOF { + err = nil + } + return + } + } +} + +func (Self *ConnCopy) CopyConn() (written int64, err error) { + defer CopyBuff.Put(Self.buf) + if Self.dst != nil && Self.src != nil { + written, err = Self.copyBuffer() + } else { + return 0, errors.New("copy conn nil src or dst") + } + if err != nil { // copyBuffer do not return io.EOF ,close all conn + logs.Warn("close by copy conn ", Self.connId, err) + if Self.dst != nil { + Self.dst.Close() + } + if Self.src != nil { + Self.src.Close() + } + } + return } //send this ip forget to get a local udp port diff --git a/lib/conn/conn.go b/lib/conn/conn.go index 7b6e729..058b7d8 100755 --- a/lib/conn/conn.go +++ b/lib/conn/conn.go @@ -10,7 +10,6 @@ import ( "github.com/cnlh/nps/lib/crypt" "github.com/cnlh/nps/lib/file" "github.com/cnlh/nps/lib/mux" - "github.com/cnlh/nps/lib/pool" "github.com/cnlh/nps/lib/rate" "github.com/cnlh/nps/vender/github.com/xtaci/kcp" "io" @@ -158,8 +157,8 @@ func (s *Conn) SendHealthInfo(info, status string) (int, error) { //get health info from conn func (s *Conn) GetHealthInfo() (info string, status bool, err error) { var l int - buf := pool.BufPoolMax.Get().([]byte) - defer pool.PutBufPoolMax(buf) + buf := common.BufPoolMax.Get().([]byte) + defer common.PutBufPoolMax(buf) if l, err = s.GetLen(); err != nil { return } else if _, err = s.ReadLen(l, buf); err != nil { @@ -232,8 +231,8 @@ func (s *Conn) SendInfo(t interface{}, flag string) (int, error) { //get task info func (s *Conn) getInfo(t interface{}) (err error) { var l int - buf := pool.BufPoolMax.Get().([]byte) - defer pool.PutBufPoolMax(buf) + buf := common.BufPoolMax.Get().([]byte) + defer common.PutBufPoolMax(buf) if l, err = s.GetLen(); err != nil { return } else if _, err = s.ReadLen(l, buf); err != nil { @@ -373,7 +372,7 @@ func CopyWaitGroup(conn1, conn2 net.Conn, crypt bool, snappy bool, rate *rate.Ra } //get crypt or snappy conn -func GetConn(conn net.Conn, cpt, snappy bool, rt *rate.Rate, isServer bool) (io.ReadWriteCloser) { +func GetConn(conn net.Conn, cpt, snappy bool, rt *rate.Rate, isServer bool) io.ReadWriteCloser { if cpt { if isServer { return rate.NewRateConn(crypt.NewTlsServerConn(conn), rt) diff --git a/lib/conn/snappy.go b/lib/conn/snappy.go index cfd33c4..cda20b5 100644 --- a/lib/conn/snappy.go +++ b/lib/conn/snappy.go @@ -1,7 +1,7 @@ package conn import ( - "github.com/cnlh/nps/lib/pool" + "github.com/cnlh/nps/lib/common" "github.com/cnlh/nps/vender/github.com/golang/snappy" "io" ) @@ -31,8 +31,8 @@ func (s *SnappyConn) Write(b []byte) (n int, err error) { //snappy压缩读 func (s *SnappyConn) Read(b []byte) (n int, err error) { - buf := pool.BufPool.Get().([]byte) - defer pool.BufPool.Put(buf) + buf := common.BufPool.Get().([]byte) + defer common.BufPool.Put(buf) if n, err = s.r.Read(buf); err != nil { return } diff --git a/lib/mux/conn.go b/lib/mux/conn.go index d30bedc..7430454 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -3,7 +3,6 @@ package mux import ( "errors" "github.com/cnlh/nps/lib/common" - "github.com/cnlh/nps/lib/pool" "github.com/cnlh/nps/vender/github.com/astaxie/beego/logs" "io" "net" @@ -27,7 +26,8 @@ type conn struct { connId int32 isClose bool readWait bool - sendClose bool + sendClose bool // MUX_CONN_CLOSE already send + writeClose bool // close conn Write hasWrite int mux *Mux } @@ -69,12 +69,14 @@ func (s *conn) Read(buf []byte) (n int, err error) { return 0, errors.New("the conn has closed") } if node, err := s.readQueue.Pop(); err != nil { + logs.Warn("conn close by read pop err", s.connId, err) s.Close() return 0, io.EOF } else { if node.val == nil { //close s.sendClose = true + logs.Warn("conn close by read ", s.connId) s.Close() return 0, io.EOF } else { @@ -90,7 +92,7 @@ func (s *conn) Read(buf []byte) (n int, err error) { } else { n = copy(buf, s.readBuffer[s.startRead:s.endRead]) s.startRead += n - pool.CopyBuff.Put(s.readBuffer) + common.CopyBuff.Put(s.readBuffer) } return } @@ -99,6 +101,12 @@ func (s *conn) Write(buf []byte) (n int, err error) { if s.isClose { return 0, errors.New("the conn has closed") } + if s.writeClose { + s.sendClose = true + logs.Warn("conn close by write ", s.connId) + s.Close() + return 0, errors.New("io: write on closed conn") + } ch := make(chan struct{}) go s.write(buf, ch) if t := s.writeTimeOut.Sub(time.Now()); t > 0 { @@ -112,19 +120,22 @@ func (s *conn) Write(buf []byte) (n int, err error) { } else { <-ch } - if s.isClose { - return 0, io.EOF - } + close(ch) + //if s.isClose { + // return 0, io.ErrClosedPipe + //} return len(buf), nil } func (s *conn) write(buf []byte, ch chan struct{}) { start := 0 l := len(buf) for { - if l-start > pool.PoolSizeCopy { - s.mux.sendInfo(common.MUX_NEW_MSG, s.connId, buf[start:start+pool.PoolSizeCopy]) - start += pool.PoolSizeCopy + if l-start > common.PoolSizeCopy { + logs.Warn("conn write > poolsizecopy") + s.mux.sendInfo(common.MUX_NEW_MSG, s.connId, buf[start:start+common.PoolSizeCopy]) + start += common.PoolSizeCopy } else { + logs.Warn("conn write <= poolsizecopy, start, len", start, l) s.mux.sendInfo(common.MUX_NEW_MSG, s.connId, buf[start:l]) break } @@ -137,21 +148,16 @@ func (s *conn) Close() (err error) { return errors.New("the conn has closed") } s.isClose = true - pool.CopyBuff.Put(s.readBuffer) + s.mux.connMap.Delete(s.connId) + common.CopyBuff.Put(s.readBuffer) if s.readWait { s.readCh <- struct{}{} } s.readQueue.Clear() - s.mux.connMap.Delete(s.connId) if !s.mux.IsClose { if !s.sendClose { - err = s.mux.sendInfo(common.MUX_CONN_CLOSE, s.connId, nil) - logs.Warn("send closing msg ok ", s.connId) - if err != nil { - logs.Warn(err) - return - } - } else { + logs.Warn("conn send close") + go s.mux.sendInfo(common.MUX_CONN_CLOSE, s.connId, nil) } } return diff --git a/lib/mux/mux.go b/lib/mux/mux.go index ad17cb0..c40427d 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -4,7 +4,6 @@ import ( "bytes" "errors" "github.com/cnlh/nps/lib/common" - "github.com/cnlh/nps/lib/pool" "github.com/cnlh/nps/vender/github.com/astaxie/beego/logs" "math" "net" @@ -42,7 +41,7 @@ func NewMux(c net.Conn, connType string) *Mux { go m.readSession() //ping go m.ping() - //go m.writeSession() + go m.writeSession() return m } @@ -53,9 +52,8 @@ func (s *Mux) NewConn() (*conn, error) { conn := NewConn(s.getId(), s) //it must be set before send s.connMap.Set(conn.connId, conn) - if err := s.sendInfo(common.MUX_NEW_CONN, conn.connId, nil); err != nil { - return nil, err - } + s.sendInfo(common.MUX_NEW_CONN, conn.connId, nil) + logs.Warn("send mux new conn ", conn.connId) //set a timer timeout 30 second timer := time.NewTimer(time.Minute * 2) defer timer.Stop() @@ -83,34 +81,41 @@ func (s *Mux) Addr() net.Addr { return s.conn.LocalAddr() } -func (s *Mux) sendInfo(flag uint8, id int32, content []byte) (err error) { +func (s *Mux) sendInfo(flag uint8, id int32, content []byte) { + var err error if flag == common.MUX_NEW_MSG { + if len(content) == 0 { + logs.Warn("send info content is nil") + } } - buf := pool.BuffPool.Get() - pack := common.MuxPackager{} + buf := common.BuffPool.Get() + //defer pool.BuffPool.Put(buf) + pack := common.MuxPack.Get() err = pack.NewPac(flag, id, content) if err != nil { s.Close() logs.Warn("new pack err", err) + common.BuffPool.Put(buf) return } err = pack.Pack(buf) if err != nil { s.Close() logs.Warn("pack err", err) + common.BuffPool.Put(buf) return } - //s.writeQueue <- buf - _, err = buf.WriteTo(s.conn) - if err != nil { - s.Close() - logs.Warn("write err", err) - } - pool.BuffPool.Put(buf) - if flag == common.MUX_CONN_CLOSE { - } - if flag == common.MUX_NEW_MSG { - } + s.writeQueue <- buf + common.MuxPack.Put(pack) + //_, err = buf.WriteTo(s.conn) + //if err != nil { + // s.Close() + // logs.Warn("write err, close mux", err) + //} + //if flag == common.MUX_CONN_CLOSE { + //} + //if flag == common.MUX_NEW_MSG { + //} return } @@ -120,7 +125,7 @@ func (s *Mux) writeSession() { buf := <-s.writeQueue l := buf.Len() n, err := buf.WriteTo(s.conn) - pool.BuffPool.Put(buf) + common.BuffPool.Put(buf) if err != nil || int(n) != l { logs.Warn("close from write to ", err, n, l) s.Close() @@ -142,7 +147,9 @@ func (s *Mux) ping() { if (math.MaxInt32 - s.id) < 10000 { s.id = 0 } - if err := s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, nil); err != nil || (s.pingOk > 10 && s.connType == "kcp") { + //logs.Warn("send mux ping") + s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, nil) + if s.pingOk > 10 && s.connType == "kcp" { s.Close() break } @@ -155,28 +162,30 @@ func (s *Mux) ping() { } func (s *Mux) readSession() { - var pack common.MuxPackager go func() { for { + pack := common.MuxPack.Get() if pack.UnPack(s.conn) != nil { break } if pack.Flag != 0 && pack.Flag != 7 { if pack.Length > 10 { - logs.Warn(pack.Flag, pack.Id, pack.Length, string(pack.Content[:10])) + //logs.Warn(pack.Flag, pack.Id, pack.Length, string(pack.Content[:10])) } } s.pingOk = 0 switch pack.Flag { case common.MUX_NEW_CONN: //new conn - //logs.Warn("mux new conn", pack.Id) + logs.Warn("mux new conn", pack.Id) conn := NewConn(pack.Id, s) s.connMap.Set(pack.Id, conn) //it has been set before send ok s.newConnCh <- conn s.sendInfo(common.MUX_NEW_CONN_OK, pack.Id, nil) + logs.Warn("send mux new conn ok", pack.Id) continue case common.MUX_PING_FLAG: //ping - s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, nil) + //logs.Warn("send mux ping return") + go s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, nil) continue case common.MUX_PING_RETURN: continue @@ -185,6 +194,7 @@ func (s *Mux) readSession() { switch pack.Flag { case common.MUX_NEW_MSG: //new msg from remote conn //insert wait queue + logs.Warn("mux new msg ", pack.Id) conn.readQueue.Push(NewBufNode(pack.Content, int(pack.Length))) //judge len if >xxx ,send stop if conn.readWait { @@ -192,21 +202,29 @@ func (s *Mux) readSession() { conn.readCh <- struct{}{} } case common.MUX_NEW_CONN_OK: //conn ok + logs.Warn("mux new conn ok ", pack.Id) conn.connStatusOkCh <- struct{}{} case common.MUX_NEW_CONN_Fail: + logs.Warn("mux new conn fail", pack.Id) conn.connStatusFailCh <- struct{}{} case common.MUX_CONN_CLOSE: //close the connection + logs.Warn("mux conn close", pack.Id) + s.connMap.Delete(pack.Id) + conn.writeClose = true conn.readQueue.Push(NewBufNode(nil, 0)) if conn.readWait { logs.Warn("close read wait", pack.Id) conn.readWait = false conn.readCh <- struct{}{} } - s.connMap.Delete(pack.Id) + logs.Warn("receive mux conn close, finish", conn.connId) } } else if pack.Flag == common.MUX_NEW_MSG { - pool.CopyBuff.Put(pack.Content) + common.CopyBuff.Put(pack.Content) + } else if pack.Flag == common.MUX_CONN_CLOSE { + logs.Warn("mux conn close no id ", pack.Id) } + common.MuxPack.Put(pack) } s.Close() }() @@ -228,7 +246,7 @@ func (s *Mux) Close() error { select { case s.closeChan <- struct{}{}: } - //s.closeChan <- struct{}{} + s.closeChan <- struct{}{} close(s.writeQueue) close(s.newConnCh) return s.conn.Close() diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index 2d3d2d0..728dfa3 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -2,9 +2,7 @@ package mux import ( "github.com/cnlh/nps/lib/common" - "github.com/cnlh/nps/lib/pool" "github.com/cnlh/nps/vender/github.com/astaxie/beego/logs" - "log" "net" "net/http" _ "net/http/pprof" @@ -27,19 +25,29 @@ func TestNewMux(t *testing.T) { go func() { m2 := NewMux(conn2, "tcp") for { + logs.Warn("npc starting accept") c, err := m2.Accept() if err != nil { - log.Fatalln(err) + logs.Warn(err) + continue } - c2, err := net.Dial("tcp", "127.0.0.1:8080") + logs.Warn("npc accept success ") + c2, err := net.Dial("tcp", "127.0.0.1:80") if err != nil { - log.Fatalln(err) + logs.Warn(err) + continue + } + var npcToServer common.ConnCopy + npcToServer.New(c2, c, 0) + go npcToServer.CopyConn() + var serverToNpc common.ConnCopy + serverToNpc.New(c, c2, 10000) + _, err = serverToNpc.CopyConn() + if err == nil { + logs.Warn("close npc") + c2.Close() + c.Close() } - go common.CopyBuffer(c2, c,0) - common.CopyBuffer(c, c2,0) - c2.Close() - c.Close() - logs.Warn("close npc") } }() @@ -47,24 +55,33 @@ func TestNewMux(t *testing.T) { m1 := NewMux(conn1, "tcp") l, err := net.Listen("tcp", "127.0.0.1:7777") if err != nil { - log.Fatalln(err) + logs.Warn(err) } for { + logs.Warn("nps starting accept") conn, err := l.Accept() if err != nil { - log.Fatalln(err) + logs.Warn(err) + continue } - + logs.Warn("nps accept success starting new conn") tmpCpnn, err := m1.NewConn() if err != nil { - log.Fatalln(err) + logs.Warn("nps new conn err ", err) + continue + } + logs.Warn("nps new conn success ", tmpCpnn.connId) + var userToNps common.ConnCopy + userToNps.New(tmpCpnn, conn, tmpCpnn.connId) + go userToNps.CopyConn() + var npsToUser common.ConnCopy + npsToUser.New(conn, tmpCpnn, tmpCpnn.connId+10000) + _, err = npsToUser.CopyConn() + if err == nil { + logs.Warn("close from out nps ", tmpCpnn.connId) + conn.Close() + tmpCpnn.Close() } - go common.CopyBuffer(tmpCpnn, conn,tmpCpnn.connId) - _, err = common.CopyBuffer(conn, tmpCpnn,tmpCpnn.connId) - logs.Warn(err, tmpCpnn.connId) - conn.Close() - tmpCpnn.Close() - logs.Warn("close from out nps ", tmpCpnn.connId) } }() @@ -77,12 +94,12 @@ func server() { var err error l, err := net.Listen("tcp", "127.0.0.1:9999") if err != nil { - log.Fatalln(err) + logs.Warn(err) } go func() { conn1, err = l.Accept() if err != nil { - log.Fatalln(err) + logs.Warn(err) } }() return @@ -92,12 +109,12 @@ func client() { var err error conn2, err = net.Dial("tcp", "127.0.0.1:9999") if err != nil { - log.Fatalln(err) + logs.Warn(err) } } func TestNewConn(t *testing.T) { - buf := pool.GetBufPoolCopy() + buf := common.GetBufPoolCopy() logs.Warn(len(buf), cap(buf)) //b := pool.GetBufPoolCopy() //b[0] = 1 diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 6a14a8d..4388fb6 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -2,7 +2,7 @@ package mux import ( "errors" - "github.com/cnlh/nps/lib/pool" + "github.com/cnlh/nps/lib/common" "sync" ) @@ -63,7 +63,7 @@ func (entry *sliceEntry) Clear() bool { return false } for i := 0; i < entry.Size(); i++ { - pool.CopyBuff.Put(entry.element[i].val) + common.CopyBuff.Put(entry.element[i].val) entry.element[i] = nil } entry.element = nil diff --git a/server/proxy/p2p.go b/server/proxy/p2p.go index 44cdea3..7c1d270 100644 --- a/server/proxy/p2p.go +++ b/server/proxy/p2p.go @@ -2,7 +2,6 @@ package proxy import ( "github.com/cnlh/nps/lib/common" - "github.com/cnlh/nps/lib/pool" "github.com/cnlh/nps/vender/github.com/astaxie/beego/logs" "net" "strings" @@ -36,7 +35,7 @@ func (s *P2PServer) Start() error { return err } for { - buf := pool.BufPoolUdp.Get().([]byte) + buf := common.BufPoolUdp.Get().([]byte) n, addr, err := s.listener.ReadFromUDP(buf) if err != nil { if strings.Contains(err.Error(), "use of closed network connection") { diff --git a/server/proxy/udp.go b/server/proxy/udp.go index 62358a4..2f88155 100755 --- a/server/proxy/udp.go +++ b/server/proxy/udp.go @@ -5,7 +5,6 @@ import ( "github.com/cnlh/nps/lib/common" "github.com/cnlh/nps/lib/conn" "github.com/cnlh/nps/lib/file" - "github.com/cnlh/nps/lib/pool" "github.com/cnlh/nps/vender/github.com/astaxie/beego/logs" "net" "strings" @@ -33,7 +32,7 @@ func (s *UdpModeServer) Start() error { if err != nil { return err } - buf := pool.BufPoolUdp.Get().([]byte) + buf := common.BufPoolUdp.Get().([]byte) for { n, addr, err := s.listener.ReadFromUDP(buf) if err != nil { @@ -59,8 +58,8 @@ func (s *UdpModeServer) process(addr *net.UDPAddr, data []byte) { return } else { s.task.Flow.Add(int64(len(data)), 0) - buf := pool.BufPoolUdp.Get().([]byte) - defer pool.BufPoolUdp.Put(buf) + buf := common.BufPoolUdp.Get().([]byte) + defer common.BufPoolUdp.Put(buf) target.Write(data) s.task.Flow.Add(int64(len(data)), 0) if n, err := target.Read(buf); err != nil { From 51a3787708bf735d24b239f2b56f44af2f2e0ee7 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sun, 1 Sep 2019 22:52:48 +0800 Subject: [PATCH 07/38] remove mux write queue, add connection close once --- lib/mux/conn.go | 46 +++++++++++++++-------------- lib/mux/mux.go | 77 ++++++++++++++++++++++++++++--------------------- 2 files changed, 69 insertions(+), 54 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 7430454..09cac16 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -6,6 +6,7 @@ import ( "github.com/cnlh/nps/vender/github.com/astaxie/beego/logs" "io" "net" + "sync" "time" ) @@ -27,9 +28,10 @@ type conn struct { isClose bool readWait bool sendClose bool // MUX_CONN_CLOSE already send - writeClose bool // close conn Write + closeFlag bool // close conn flag hasWrite int mux *Mux + once sync.Once } func NewConn(connId int32, mux *Mux) *conn { @@ -41,6 +43,7 @@ func NewConn(connId int32, mux *Mux) *conn { readQueue: NewQueue(), connId: connId, mux: mux, + once: sync.Once{}, } return c } @@ -72,18 +75,14 @@ func (s *conn) Read(buf []byte) (n int, err error) { logs.Warn("conn close by read pop err", s.connId, err) s.Close() return 0, io.EOF + } else if node.val == nil { + s.sendClose = true + logs.Warn("conn close by read ", s.connId) + s.Close() } else { - if node.val == nil { - //close - s.sendClose = true - logs.Warn("conn close by read ", s.connId) - s.Close() - return 0, io.EOF - } else { - s.readBuffer = node.val - s.endRead = node.l - s.startRead = 0 - } + s.readBuffer = node.val + s.endRead = node.l + s.startRead = 0 } } if len(buf) < s.endRead-s.startRead { @@ -101,7 +100,7 @@ func (s *conn) Write(buf []byte) (n int, err error) { if s.isClose { return 0, errors.New("the conn has closed") } - if s.writeClose { + if s.closeFlag { s.sendClose = true logs.Warn("conn close by write ", s.connId) s.Close() @@ -131,11 +130,11 @@ func (s *conn) write(buf []byte, ch chan struct{}) { l := len(buf) for { if l-start > common.PoolSizeCopy { - logs.Warn("conn write > poolsizecopy") + //logs.Warn("conn write > poolsizecopy") s.mux.sendInfo(common.MUX_NEW_MSG, s.connId, buf[start:start+common.PoolSizeCopy]) start += common.PoolSizeCopy } else { - logs.Warn("conn write <= poolsizecopy, start, len", start, l) + //logs.Warn("conn write <= poolsizecopy, start, len", start, l) s.mux.sendInfo(common.MUX_NEW_MSG, s.connId, buf[start:l]) break } @@ -144,20 +143,25 @@ func (s *conn) write(buf []byte, ch chan struct{}) { } func (s *conn) Close() (err error) { + s.once.Do(s.closeProcess) + return +} + +func (s *conn) closeProcess() { if s.isClose { - return errors.New("the conn has closed") + logs.Warn("has closed ", s.connId) + return } s.isClose = true + s.readWait = false s.mux.connMap.Delete(s.connId) common.CopyBuff.Put(s.readBuffer) - if s.readWait { - s.readCh <- struct{}{} - } + close(s.readCh) s.readQueue.Clear() if !s.mux.IsClose { if !s.sendClose { - logs.Warn("conn send close") - go s.mux.sendInfo(common.MUX_CONN_CLOSE, s.connId, nil) + logs.Warn("conn send close", s.connId) + s.mux.sendInfo(common.MUX_CONN_CLOSE, s.connId, nil) } } return diff --git a/lib/mux/mux.go b/lib/mux/mux.go index c40427d..1b90c60 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -41,7 +41,7 @@ func NewMux(c net.Conn, connType string) *Mux { go m.readSession() //ping go m.ping() - go m.writeSession() + //go m.writeSession() return m } @@ -68,7 +68,7 @@ func (s *Mux) NewConn() (*conn, error) { func (s *Mux) Accept() (net.Conn, error) { if s.IsClose { - return nil, errors.New("accpet error,the conn has closed") + return nil, errors.New("accpet error,the mux has closed") } conn := <-s.newConnCh if conn == nil { @@ -91,31 +91,29 @@ func (s *Mux) sendInfo(flag uint8, id int32, content []byte) { buf := common.BuffPool.Get() //defer pool.BuffPool.Put(buf) pack := common.MuxPack.Get() + defer common.MuxPack.Put(pack) err = pack.NewPac(flag, id, content) if err != nil { - s.Close() logs.Warn("new pack err", err) common.BuffPool.Put(buf) return } err = pack.Pack(buf) if err != nil { - s.Close() logs.Warn("pack err", err) common.BuffPool.Put(buf) return } - s.writeQueue <- buf - common.MuxPack.Put(pack) - //_, err = buf.WriteTo(s.conn) - //if err != nil { - // s.Close() - // logs.Warn("write err, close mux", err) - //} - //if flag == common.MUX_CONN_CLOSE { - //} - //if flag == common.MUX_NEW_MSG { - //} + if pack.Flag == common.MUX_NEW_CONN { + logs.Warn("sendinfo mux new conn, insert to write queue", pack.Id) + } + //s.writeQueue <- buf + _, err = buf.WriteTo(s.conn) + if err != nil { + s.Close() + logs.Warn("write err, close mux", err) + } + common.BuffPool.Put(buf) return } @@ -127,7 +125,7 @@ func (s *Mux) writeSession() { n, err := buf.WriteTo(s.conn) common.BuffPool.Put(buf) if err != nil || int(n) != l { - logs.Warn("close from write to ", err, n, l) + logs.Warn("close from write session fail ", err, n, l) s.Close() break } @@ -163,8 +161,8 @@ func (s *Mux) ping() { func (s *Mux) readSession() { go func() { + pack := common.MuxPack.Get() for { - pack := common.MuxPack.Get() if pack.UnPack(s.conn) != nil { break } @@ -173,10 +171,13 @@ func (s *Mux) readSession() { //logs.Warn(pack.Flag, pack.Id, pack.Length, string(pack.Content[:10])) } } + if pack.Flag == common.MUX_NEW_CONN { + logs.Warn("unpack mux new conn", pack.Id) + } s.pingOk = 0 switch pack.Flag { case common.MUX_NEW_CONN: //new conn - logs.Warn("mux new conn", pack.Id) + logs.Warn("rec mux new conn", pack.Id) conn := NewConn(pack.Id, s) s.connMap.Set(pack.Id, conn) //it has been set before send ok s.newConnCh <- conn @@ -194,38 +195,48 @@ func (s *Mux) readSession() { switch pack.Flag { case common.MUX_NEW_MSG: //new msg from remote conn //insert wait queue - logs.Warn("mux new msg ", pack.Id) - conn.readQueue.Push(NewBufNode(pack.Content, int(pack.Length))) + buf := common.CopyBuff.Get() + buf = pack.Content + logs.Warn("rec mux new msg ", pack.Id, string(buf[0:15])) + conn.readQueue.Push(NewBufNode(buf, int(pack.Length))) //judge len if >xxx ,send stop if conn.readWait { conn.readWait = false conn.readCh <- struct{}{} } + continue case common.MUX_NEW_CONN_OK: //conn ok - logs.Warn("mux new conn ok ", pack.Id) + logs.Warn("rec mux new conn ok ", pack.Id) conn.connStatusOkCh <- struct{}{} + continue case common.MUX_NEW_CONN_Fail: - logs.Warn("mux new conn fail", pack.Id) + logs.Warn("rec mux new conn fail", pack.Id) conn.connStatusFailCh <- struct{}{} + continue case common.MUX_CONN_CLOSE: //close the connection - logs.Warn("mux conn close", pack.Id) + logs.Warn("rec mux conn close", pack.Id) s.connMap.Delete(pack.Id) - conn.writeClose = true - conn.readQueue.Push(NewBufNode(nil, 0)) - if conn.readWait { - logs.Warn("close read wait", pack.Id) - conn.readWait = false - conn.readCh <- struct{}{} + conn.closeFlag = true + conn.sendClose = true + if !conn.isClose { + conn.readQueue.Push(NewBufNode(nil, 0)) + if conn.readWait { + logs.Warn("mux conn close read wait", pack.Id) + conn.readWait = false + conn.readCh <- struct{}{} + logs.Warn("mux conn close read wait pass", pack.Id) + } } logs.Warn("receive mux conn close, finish", conn.connId) + continue } - } else if pack.Flag == common.MUX_NEW_MSG { - common.CopyBuff.Put(pack.Content) } else if pack.Flag == common.MUX_CONN_CLOSE { - logs.Warn("mux conn close no id ", pack.Id) + logs.Warn("rec mux conn close no id ", pack.Id) + continue } - common.MuxPack.Put(pack) } + common.MuxPack.Put(pack) + logs.Warn("read session put pack ", pack.Id) s.Close() }() select { From 3413ceb7c21a343ced1c45295b20361d9eb18fcf Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Mon, 2 Sep 2019 00:18:52 +0800 Subject: [PATCH 08/38] add write queue again --- lib/mux/conn.go | 5 ----- lib/mux/mux.go | 16 ++++++++-------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 09cac16..494fd4e 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -148,12 +148,7 @@ func (s *conn) Close() (err error) { } func (s *conn) closeProcess() { - if s.isClose { - logs.Warn("has closed ", s.connId) - return - } s.isClose = true - s.readWait = false s.mux.connMap.Delete(s.connId) common.CopyBuff.Put(s.readBuffer) close(s.readCh) diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 1b90c60..8673c24 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -41,7 +41,7 @@ func NewMux(c net.Conn, connType string) *Mux { go m.readSession() //ping go m.ping() - //go m.writeSession() + go m.writeSession() return m } @@ -107,13 +107,13 @@ func (s *Mux) sendInfo(flag uint8, id int32, content []byte) { if pack.Flag == common.MUX_NEW_CONN { logs.Warn("sendinfo mux new conn, insert to write queue", pack.Id) } - //s.writeQueue <- buf - _, err = buf.WriteTo(s.conn) - if err != nil { - s.Close() - logs.Warn("write err, close mux", err) - } - common.BuffPool.Put(buf) + s.writeQueue <- buf + //_, err = buf.WriteTo(s.conn) + //if err != nil { + // s.Close() + // logs.Warn("write err, close mux", err) + //} + //common.BuffPool.Put(buf) return } From 9d3df6be7e0e623819463df0d4eb2940b9319d96 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sun, 8 Sep 2019 23:49:16 +0800 Subject: [PATCH 09/38] add mux slide window --- lib/common/netpackager.go | 20 +- lib/common/pool.go | 60 ++++-- lib/common/util.go | 109 ++--------- lib/mux/conn.go | 398 ++++++++++++++++++++++++++++++-------- lib/mux/mux.go | 90 +++++---- lib/mux/mux_test.go | 39 ++-- 6 files changed, 474 insertions(+), 242 deletions(-) diff --git a/lib/common/netpackager.go b/lib/common/netpackager.go index 6d67f44..f2ddaec 100644 --- a/lib/common/netpackager.go +++ b/lib/common/netpackager.go @@ -150,8 +150,9 @@ func (Self *ConnPackager) UnPack(reader io.Reader) (err error) { } type MuxPackager struct { - Flag uint8 - Id int32 + Flag uint8 + Id int32 + Window uint16 BasePackager } @@ -161,6 +162,15 @@ func (Self *MuxPackager) NewPac(flag uint8, id int32, content ...interface{}) (e if flag == MUX_NEW_MSG { err = Self.BasePackager.NewPac(content...) } + if flag == MUX_MSG_SEND_OK { + // MUX_MSG_SEND_OK only allows one data + switch content[0].(type) { + case int: + Self.Window = uint16(content[0].(int)) + case uint16: + Self.Window = content[0].(uint16) + } + } return } @@ -176,6 +186,9 @@ func (Self *MuxPackager) Pack(writer io.Writer) (err error) { if Self.Flag == MUX_NEW_MSG { err = Self.BasePackager.Pack(writer) } + if Self.Flag == MUX_MSG_SEND_OK { + err = binary.Write(writer, binary.LittleEndian, Self.Window) + } return } @@ -192,5 +205,8 @@ func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { if Self.Flag == MUX_NEW_MSG { err = Self.BasePackager.UnPack(reader) } + if Self.Flag == MUX_MSG_SEND_OK { + err = binary.Read(reader, binary.LittleEndian, &Self.Window) + } return } diff --git a/lib/common/pool.go b/lib/common/pool.go index 205378d..98da5d3 100644 --- a/lib/common/pool.go +++ b/lib/common/pool.go @@ -9,6 +9,7 @@ const PoolSize = 64 * 1024 const PoolSizeSmall = 100 const PoolSizeUdp = 1472 const PoolSizeCopy = 32 << 10 +const PoolSizeWindow = 1<<16 - 1 var BufPool = sync.Pool{ New: func() interface{} { @@ -59,11 +60,11 @@ func PutBufPoolMax(buf []byte) { } } -type CopyBufferPool struct { +type copyBufferPool struct { pool sync.Pool } -func (Self *CopyBufferPool) New() { +func (Self *copyBufferPool) New() { Self.pool = sync.Pool{ New: func() interface{} { return make([]byte, PoolSizeCopy, PoolSizeCopy) @@ -71,24 +72,49 @@ func (Self *CopyBufferPool) New() { } } -func (Self *CopyBufferPool) Get() []byte { +func (Self *copyBufferPool) Get() []byte { buf := Self.pool.Get().([]byte) return buf[:PoolSizeCopy] // just like make a new slice, but data may not be 0 } -func (Self *CopyBufferPool) Put(x []byte) { +func (Self *copyBufferPool) Put(x []byte) { if len(x) == PoolSizeCopy { Self.pool.Put(x) } else { - x = nil // buf is not full, maybe truncated by gc in pool, not allowed + x = nil // buf is not full, not allowed, New method returns a full buf } } -type BufferPool struct { +type windowBufferPool struct { pool sync.Pool } -func (Self *BufferPool) New() { +func (Self *windowBufferPool) New() { + Self.pool = sync.Pool{ + New: func() interface{} { + return make([]byte, 0, PoolSizeWindow) + }, + } +} + +func (Self *windowBufferPool) Get() (buf []byte) { + buf = Self.pool.Get().([]byte) + return buf[:0] +} + +func (Self *windowBufferPool) Put(x []byte) { + if cap(x) == PoolSizeWindow { + Self.pool.Put(x[:PoolSizeWindow]) // make buf to full + } else { + x = nil + } +} + +type bufferPool struct { + pool sync.Pool +} + +func (Self *bufferPool) New() { Self.pool = sync.Pool{ New: func() interface{} { return new(bytes.Buffer) @@ -96,20 +122,20 @@ func (Self *BufferPool) New() { } } -func (Self *BufferPool) Get() *bytes.Buffer { +func (Self *bufferPool) Get() *bytes.Buffer { return Self.pool.Get().(*bytes.Buffer) } -func (Self *BufferPool) Put(x *bytes.Buffer) { +func (Self *bufferPool) Put(x *bytes.Buffer) { x.Reset() Self.pool.Put(x) } -type MuxPackagerPool struct { +type muxPackagerPool struct { pool sync.Pool } -func (Self *MuxPackagerPool) New() { +func (Self *muxPackagerPool) New() { Self.pool = sync.Pool{ New: func() interface{} { pack := MuxPackager{} @@ -118,27 +144,29 @@ func (Self *MuxPackagerPool) New() { } } -func (Self *MuxPackagerPool) Get() *MuxPackager { +func (Self *muxPackagerPool) Get() *MuxPackager { pack := Self.pool.Get().(*MuxPackager) buf := CopyBuff.Get() pack.Content = buf return pack } -func (Self *MuxPackagerPool) Put(pack *MuxPackager) { +func (Self *muxPackagerPool) Put(pack *MuxPackager) { CopyBuff.Put(pack.Content) Self.pool.Put(pack) } var once = sync.Once{} -var BuffPool = BufferPool{} -var CopyBuff = CopyBufferPool{} -var MuxPack = MuxPackagerPool{} +var BuffPool = bufferPool{} +var CopyBuff = copyBufferPool{} +var MuxPack = muxPackagerPool{} +var WindowBuff = windowBufferPool{} func newPool() { BuffPool.New() CopyBuff.New() MuxPack.New() + WindowBuff.New() } func init() { diff --git a/lib/common/util.go b/lib/common/util.go index 3b4369f..60b01dc 100755 --- a/lib/common/util.go +++ b/lib/common/util.go @@ -4,9 +4,7 @@ import ( "bytes" "encoding/base64" "encoding/binary" - "errors" "github.com/cnlh/nps/lib/crypt" - "github.com/cnlh/nps/vender/github.com/astaxie/beego/logs" "html/template" "io" "io/ioutil" @@ -264,96 +262,31 @@ func GetPortByAddr(addr string) int { return p } -type ConnCopy struct { - dst net.Conn - src net.Conn - buf []byte - connId int32 -} - -func (Self *ConnCopy) New(dst net.Conn, src net.Conn, connId int32) { - Self.dst = dst - Self.src = src - Self.buf = CopyBuff.Get() - Self.connId = connId -} - -func (Self *ConnCopy) copyBufferOnce() (written int64, err error) { - nr, er := Self.src.Read(Self.buf) - if nr > 0 { - //logs.Warn("write", Self.connId, nr, string(buf[0:10])) - nw, ew := Self.dst.Write(Self.buf[0:nr]) - if nw > 0 { - written = int64(nw) - } - if ew != nil { - //logs.Warn("write err ew id nw", ew, Self.connId, nw) - err = ew - return - } - if nr != nw { - err = io.ErrShortWrite - return - } - if nw == 0 { - err = errors.New("io: write on closed pipe") - //logs.Warn("write buffer", err) - return - } - } - if nr == 0 && er == nil { - err = errors.New("io: read on closed pipe") - //logs.Warn("read buffer", err) - return - } - if er != nil { - err = er - return - } - return -} - -func (Self *ConnCopy) copyBuffer() (written int64, err error) { - var write int64 - write, err = Self.copyBufferOnce() // first copy, if written is zero and err is io.EOF - // means conn already closed, so need to close all the conn - written += write - if err == io.EOF && written == 0 { - err = errors.New("io: read on closed pipe") - return - } else if err == io.EOF && written > 0 { - err = nil - return - } +func CopyBuffer(dst io.Writer, src io.Reader) (written int64, err error) { + buf := CopyBuff.Get() + defer CopyBuff.Put(buf) for { - write, err = Self.copyBufferOnce() - written += write - if err != nil { - if err == io.EOF { - err = nil + nr, er := src.Read(buf) + if nr > 0 { + nw, ew := dst.Write(buf[0:nr]) + if nw > 0 { + written += int64(nw) } - return + if ew != nil { + err = ew + break + } + if nr != nw { + err = io.ErrShortWrite + break + } + } + if er != nil { + err = er + break } } -} - -func (Self *ConnCopy) CopyConn() (written int64, err error) { - defer CopyBuff.Put(Self.buf) - if Self.dst != nil && Self.src != nil { - written, err = Self.copyBuffer() - } else { - return 0, errors.New("copy conn nil src or dst") - } - if err != nil { // copyBuffer do not return io.EOF ,close all conn - logs.Warn("close by copy conn ", Self.connId, err) - if Self.dst != nil { - Self.dst.Close() - } - if Self.src != nil { - Self.src.Close() - } - } - return + return written, err } //send this ip forget to get a local udp port diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 494fd4e..08b0db0 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -17,34 +17,37 @@ type conn struct { connStatusFailCh chan struct{} readTimeOut time.Time writeTimeOut time.Time - readBuffer []byte - startRead int //now read position - endRead int //now end read - readFlag bool - readCh chan struct{} - readQueue *sliceEntry - stopWrite bool - connId int32 - isClose bool - readWait bool - sendClose bool // MUX_CONN_CLOSE already send - closeFlag bool // close conn flag - hasWrite int - mux *Mux - once sync.Once + //readBuffer []byte + //startRead int //now read position + //endRead int //now end read + //readFlag bool + //readCh chan struct{} + //readQueue *sliceEntry + //stopWrite bool + connId int32 + isClose bool + //readWait bool + closeFlag bool // close conn flag + hasWrite int + receiveWindow *window + sendWindow *window + mux *Mux + once sync.Once } func NewConn(connId int32, mux *Mux) *conn { c := &conn{ - readCh: make(chan struct{}), getStatusCh: make(chan struct{}), connStatusOkCh: make(chan struct{}), connStatusFailCh: make(chan struct{}), - readQueue: NewQueue(), connId: connId, + receiveWindow: new(window), + sendWindow: new(window), mux: mux, once: sync.Once{}, } + c.receiveWindow.NewReceive() + c.sendWindow.NewSend() return c } @@ -52,94 +55,99 @@ func (s *conn) Read(buf []byte) (n int, err error) { if s.isClose || buf == nil { return 0, errors.New("the conn has closed") } - if s.endRead-s.startRead == 0 { //read finish or start - if s.readQueue.Size() == 0 { - s.readWait = true - if t := s.readTimeOut.Sub(time.Now()); t > 0 { - timer := time.NewTimer(t) - defer timer.Stop() - select { - case <-timer.C: - s.readWait = false - return 0, errors.New("read timeout") - case <-s.readCh: - } - } else { - <-s.readCh - } + nCh := make(chan int) + errCh := make(chan error) + defer close(nCh) + defer close(errCh) + // waiting for takeout from receive window finish or timeout + go s.readWindow(buf, nCh, errCh) + if t := s.readTimeOut.Sub(time.Now()); t > 0 { + timer := time.NewTimer(t) + defer timer.Stop() + select { + case <-timer.C: + return 0, errors.New("read timeout") + case n = <-nCh: + err = <-errCh } - if s.isClose { //If the connection is closed instead of continuing command - return 0, errors.New("the conn has closed") - } - if node, err := s.readQueue.Pop(); err != nil { - logs.Warn("conn close by read pop err", s.connId, err) - s.Close() - return 0, io.EOF - } else if node.val == nil { - s.sendClose = true - logs.Warn("conn close by read ", s.connId) - s.Close() - } else { - s.readBuffer = node.val - s.endRead = node.l - s.startRead = 0 - } - } - if len(buf) < s.endRead-s.startRead { - n = copy(buf, s.readBuffer[s.startRead:s.startRead+len(buf)]) - s.startRead += n } else { - n = copy(buf, s.readBuffer[s.startRead:s.endRead]) - s.startRead += n - common.CopyBuff.Put(s.readBuffer) + n = <-nCh + err = <-errCh } + logs.Warn("read window finish conn read n err buf", n, err, string(buf[:15])) return } +func (s *conn) readWindow(buf []byte, nCh chan int, errCh chan error) { + n, err := s.receiveWindow.Read(buf) + //logs.Warn("readwindow goroutine status n err buf", n, err, string(buf[:15])) + if s.receiveWindow.WindowFull { + if s.receiveWindow.Size() > 0 { + // window.Read may be invoked before window.Write, and WindowFull flag change to true + // so make sure that receiveWindow is free some space + s.receiveWindow.WindowFull = false + logs.Warn("defer send mux msg send ok size", s.receiveWindow.Size()) + s.mux.sendInfo(common.MUX_MSG_SEND_OK, s.connId, s.receiveWindow.Size()) + // acknowledge other side, have empty some receive window space + } + } + nCh <- n + errCh <- err +} + func (s *conn) Write(buf []byte) (n int, err error) { if s.isClose { return 0, errors.New("the conn has closed") } if s.closeFlag { - s.sendClose = true logs.Warn("conn close by write ", s.connId) - s.Close() + //s.Close() return 0, errors.New("io: write on closed conn") } - ch := make(chan struct{}) - go s.write(buf, ch) + + nCh := make(chan int) + errCh := make(chan error) + defer close(nCh) + defer close(errCh) + s.sendWindow.SetSendBuf(buf) // set the buf to send window + go s.write(nCh, errCh) + // waiting for send to other side or timeout if t := s.writeTimeOut.Sub(time.Now()); t > 0 { timer := time.NewTimer(t) defer timer.Stop() select { case <-timer.C: return 0, errors.New("write timeout") - case <-ch: + case n = <-nCh: + err = <-errCh } } else { - <-ch + n = <-nCh + err = <-errCh } - close(ch) - //if s.isClose { - // return 0, io.ErrClosedPipe - //} - return len(buf), nil + return } -func (s *conn) write(buf []byte, ch chan struct{}) { - start := 0 - l := len(buf) +func (s *conn) write(nCh chan int, errCh chan error) { + var n int + var err error for { - if l-start > common.PoolSizeCopy { - //logs.Warn("conn write > poolsizecopy") - s.mux.sendInfo(common.MUX_NEW_MSG, s.connId, buf[start:start+common.PoolSizeCopy]) - start += common.PoolSizeCopy - } else { - //logs.Warn("conn write <= poolsizecopy, start, len", start, l) - s.mux.sendInfo(common.MUX_NEW_MSG, s.connId, buf[start:l]) + buf, err := s.sendWindow.WriteTo() + // get the usable window size buf from send window + if buf == nil && err == io.EOF { + // send window is drain, break the loop + err = nil break } + if err != nil { + break + } + n += len(buf) + //logs.Warn("send window buf len", len(buf)) + s.mux.sendInfo(common.MUX_NEW_MSG, s.connId, buf) + // send to other side, not send nil data to other side } - ch <- struct{}{} + nCh <- n + errCh <- err } func (s *conn) Close() (err error) { @@ -150,15 +158,14 @@ func (s *conn) Close() (err error) { func (s *conn) closeProcess() { s.isClose = true s.mux.connMap.Delete(s.connId) - common.CopyBuff.Put(s.readBuffer) - close(s.readCh) - s.readQueue.Clear() if !s.mux.IsClose { - if !s.sendClose { - logs.Warn("conn send close", s.connId) - s.mux.sendInfo(common.MUX_CONN_CLOSE, s.connId, nil) - } + logs.Warn("conn send close", s.connId) + // if server or user close the conn while reading, will get a io.EOF + // and this Close method will be invoke, send this signal to close other side + s.mux.sendInfo(common.MUX_CONN_CLOSE, s.connId, nil) } + s.sendWindow.CloseWindow() + s.receiveWindow.CloseWindow() return } @@ -185,3 +192,226 @@ func (s *conn) SetWriteDeadline(t time.Time) error { s.writeTimeOut = t return nil } + +type window struct { + windowBuff []byte + off uint16 + readOp chan struct{} + readWait bool + WindowFull bool + usableReceiveWindow chan uint16 + WriteWg sync.WaitGroup + closeOp bool + closeOpCh chan struct{} + WriteEndOp chan struct{} + mutex sync.Mutex +} + +func (Self *window) NewReceive() { + // initial a window for receive + Self.windowBuff = common.WindowBuff.Get() + Self.readOp = make(chan struct{}) + Self.WriteEndOp = make(chan struct{}) + Self.closeOpCh = make(chan struct{}, 2) +} + +func (Self *window) NewSend() { + // initial a window for send + Self.usableReceiveWindow = make(chan uint16) + Self.closeOpCh = make(chan struct{}, 2) +} + +func (Self *window) SetSendBuf(buf []byte) { + // send window buff from conn write method, set it to send window + Self.mutex.Lock() + Self.windowBuff = buf + Self.off = 0 + Self.mutex.Unlock() +} + +func (Self *window) fullSlide() { + // slide by allocate + newBuf := common.WindowBuff.Get() + copy(newBuf[0:Self.len()], Self.windowBuff[Self.off:]) + Self.off = 0 + common.WindowBuff.Put(Self.windowBuff) + Self.windowBuff = newBuf + return +} + +func (Self *window) liteSlide() { + // slide by re slice + Self.windowBuff = Self.windowBuff[Self.off:] + Self.off = 0 + return +} + +func (Self *window) Size() (n int) { + // receive Window remaining + n = common.PoolSizeWindow - Self.len() + return +} + +func (Self *window) len() (n int) { + n = len(Self.windowBuff[Self.off:]) + return +} + +func (Self *window) cap() (n int) { + n = cap(Self.windowBuff[Self.off:]) + return +} + +func (Self *window) grow(n int) { + Self.windowBuff = Self.windowBuff[:Self.len()+n] +} + +func (Self *window) Write(p []byte) (n int, err error) { + if Self.closeOp { + logs.Warn("window write closed len p", len(p)) + return 0, errors.New("conn.receiveWindow: write on closed window") + } + if len(p) > Self.Size() { + return 0, errors.New("conn.receiveWindow: write too large") + } + if Self.readWait { + defer Self.allowRead() + } + //logs.Warn("window write p string", len(p), string(p[:15])) + Self.mutex.Lock() + // slide the offset + if len(p) > Self.cap()-Self.len() { + // not enough space, need to allocate + Self.fullSlide() + //logs.Warn("window write full slide len cap", Self.len(), Self.cap()) + } else { + // have enough space, re slice + Self.liteSlide() + //logs.Warn("window write lite slide len cap", Self.len(), Self.cap()) + } + length := Self.len() // length before grow + Self.grow(len(p)) // grow for copy + n = copy(Self.windowBuff[length:], p) // must copy data before allow Read + //logs.Warn("window write copy n len cap buf", n, Self.len(), Self.cap(), string(Self.windowBuff[Self.len()-n:Self.len()+15-n])) + Self.mutex.Unlock() + return n, nil +} + +func (Self *window) allowRead() (closed bool) { + //logs.Warn("length 0 read op") + Self.readWait = false + if Self.closeOp { + close(Self.readOp) + return true + } + select { + case <-Self.closeOpCh: + close(Self.readOp) + return true + case Self.readOp <- struct{}{}: + //logs.Warn("length 0 read op finish") + return false + } +} + +func (Self *window) Read(p []byte) (n int, err error) { + //logs.Warn("starting window read method len ", Self.len()) + if Self.closeOp { + return 0, io.EOF // Write method receive close signal, returns eof + } + if Self.len() == 0 { + // window is empty, waiting for Write method send a success readOp signal + // or get timeout or close + Self.readWait = true + ticker := time.NewTicker(2 * time.Minute) + defer ticker.Stop() + select { + case _, ok := <-Self.readOp: + //logs.Warn("read window read op len cap", Self.len(), Self.cap()) + if !ok { + return 0, errors.New("conn.receiveWindow: window closed") + } + case <-Self.WriteEndOp: + return 0, io.EOF // receive eof signal, returns eof + case <-ticker.C: + return 0, errors.New("conn.receiveWindow: read time out") + case <-Self.closeOpCh: + close(Self.readOp) + return 0, io.EOF // receive close signal, returns eof + } + } + //logs.Warn("window read start len window buff", Self.len(), string(Self.windowBuff[Self.off:Self.off+15])) + Self.mutex.Lock() + n = copy(p, Self.windowBuff[Self.off:]) + Self.off += uint16(n) + p = p[:n] + //logs.Warn("window read finish n len p p", n, len(p), string(p[:15])) + Self.mutex.Unlock() + return +} + +func (Self *window) WriteTo() (p []byte, err error) { + if Self.closeOp { + logs.Warn("window write to closed") + return nil, errors.New("conn.writeWindow: window closed") + } + if Self.len() == 0 { + return nil, io.EOF + // send window buff is drain, return eof and get another one + } + var windowSize uint16 + var ok bool +waiting: + ticker := time.NewTicker(2 * time.Minute) + defer ticker.Stop() + // waiting for receive usable window size, or timeout + select { + case windowSize, ok = <-Self.usableReceiveWindow: + if !ok { + return nil, errors.New("conn.writeWindow: window closed") + } + case <-ticker.C: + return nil, errors.New("conn.writeWindow: write to time out") + } + if windowSize == 0 { + goto waiting // waiting for another usable window size + } + Self.mutex.Lock() + if windowSize > uint16(Self.len()) { + // usable window size is bigger than window buff size, send the full buff + //logs.Warn("window size overflow windowSize len()", windowSize, Self.len()) + windowSize = uint16(Self.len()) + } + //logs.Warn("window buff off windowSize", Self.off, windowSize) + p = Self.windowBuff[Self.off : windowSize+Self.off] + Self.off += windowSize + Self.mutex.Unlock() + return +} + +func (Self *window) SetAllowSize(value uint16) (closed bool) { + defer func() { + if recover() != nil { + closed = true + } + }() + if Self.closeOp { + close(Self.usableReceiveWindow) + return true + } + select { + case Self.usableReceiveWindow <- value: + return false + case <-Self.closeOpCh: + close(Self.usableReceiveWindow) + return true + } +} + +func (Self *window) CloseWindow() { + Self.closeOp = true + Self.closeOpCh <- struct{}{} + Self.closeOpCh <- struct{}{} + close(Self.closeOpCh) + return +} diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 8673c24..9310c64 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -81,10 +81,10 @@ func (s *Mux) Addr() net.Addr { return s.conn.LocalAddr() } -func (s *Mux) sendInfo(flag uint8, id int32, content []byte) { +func (s *Mux) sendInfo(flag uint8, id int32, data interface{}) { var err error if flag == common.MUX_NEW_MSG { - if len(content) == 0 { + if len(data.([]byte)) == 0 { logs.Warn("send info content is nil") } } @@ -92,7 +92,7 @@ func (s *Mux) sendInfo(flag uint8, id int32, content []byte) { //defer pool.BuffPool.Put(buf) pack := common.MuxPack.Get() defer common.MuxPack.Put(pack) - err = pack.NewPac(flag, id, content) + err = pack.NewPac(flag, id, data) if err != nil { logs.Warn("new pack err", err) common.BuffPool.Put(buf) @@ -163,6 +163,7 @@ func (s *Mux) readSession() { go func() { pack := common.MuxPack.Get() for { + pack = common.MuxPack.Get() if pack.UnPack(s.conn) != nil { break } @@ -172,17 +173,18 @@ func (s *Mux) readSession() { } } if pack.Flag == common.MUX_NEW_CONN { - logs.Warn("unpack mux new conn", pack.Id) + logs.Warn("unpack mux new connection", pack.Id) } s.pingOk = 0 switch pack.Flag { - case common.MUX_NEW_CONN: //new conn - logs.Warn("rec mux new conn", pack.Id) + case common.MUX_NEW_CONN: //new connection + logs.Warn("rec mux new connection", pack.Id) conn := NewConn(pack.Id, s) s.connMap.Set(pack.Id, conn) //it has been set before send ok s.newConnCh <- conn + go conn.sendWindow.SetAllowSize(512) // set the initial receive window s.sendInfo(common.MUX_NEW_CONN_OK, pack.Id, nil) - logs.Warn("send mux new conn ok", pack.Id) + logs.Warn("send mux new connection ok", pack.Id) continue case common.MUX_PING_FLAG: //ping //logs.Warn("send mux ping return") @@ -191,49 +193,65 @@ func (s *Mux) readSession() { case common.MUX_PING_RETURN: continue } - if conn, ok := s.connMap.Get(pack.Id); ok && !conn.isClose { + if connection, ok := s.connMap.Get(pack.Id); ok && !connection.isClose { switch pack.Flag { - case common.MUX_NEW_MSG: //new msg from remote conn + case common.MUX_NEW_MSG: //new msg from remote connection //insert wait queue - buf := common.CopyBuff.Get() - buf = pack.Content - logs.Warn("rec mux new msg ", pack.Id, string(buf[0:15])) - conn.readQueue.Push(NewBufNode(buf, int(pack.Length))) - //judge len if >xxx ,send stop - if conn.readWait { - conn.readWait = false - conn.readCh <- struct{}{} + if connection.isClose { + logs.Warn("rec mux new msg closed", pack.Id, string(pack.Content[0:15])) + continue } + connection.receiveWindow.WriteWg.Add(1) + logs.Warn("rec mux new msg ", pack.Id, string(pack.Content[0:15])) + go func(connection *conn, content []byte) { // do not block read session + _, err := connection.receiveWindow.Write(content) + if err != nil { + logs.Warn("mux new msg err close", err) + s.Close() + } + size := connection.receiveWindow.Size() + if size == 0 { + connection.receiveWindow.WindowFull = true + } + s.sendInfo(common.MUX_MSG_SEND_OK, connection.connId, size) + logs.Warn("send mux new msg ok", pack.Id, size) + connection.receiveWindow.WriteWg.Done() + }(connection, pack.Content) continue - case common.MUX_NEW_CONN_OK: //conn ok - logs.Warn("rec mux new conn ok ", pack.Id) - conn.connStatusOkCh <- struct{}{} + case common.MUX_NEW_CONN_OK: //connection ok + logs.Warn("rec mux new connection ok ", pack.Id) + connection.connStatusOkCh <- struct{}{} + go connection.sendWindow.SetAllowSize(512) + // set the initial receive window both side continue case common.MUX_NEW_CONN_Fail: - logs.Warn("rec mux new conn fail", pack.Id) - conn.connStatusFailCh <- struct{}{} + logs.Warn("rec mux new connection fail", pack.Id) + connection.connStatusFailCh <- struct{}{} + continue + case common.MUX_MSG_SEND_OK: + if connection.isClose { + logs.Warn("rec mux msg send ok id window closed!", pack.Id, pack.Window) + continue + } + logs.Warn("rec mux msg send ok id window", pack.Id, pack.Window) + go connection.sendWindow.SetAllowSize(pack.Window) continue case common.MUX_CONN_CLOSE: //close the connection - logs.Warn("rec mux conn close", pack.Id) + logs.Warn("rec mux connection close", pack.Id) s.connMap.Delete(pack.Id) - conn.closeFlag = true - conn.sendClose = true - if !conn.isClose { - conn.readQueue.Push(NewBufNode(nil, 0)) - if conn.readWait { - logs.Warn("mux conn close read wait", pack.Id) - conn.readWait = false - conn.readCh <- struct{}{} - logs.Warn("mux conn close read wait pass", pack.Id) - } - } - logs.Warn("receive mux conn close, finish", conn.connId) + connection.closeFlag = true + go func(connection *conn) { + connection.receiveWindow.WriteWg.Wait() + connection.receiveWindow.WriteEndOp <- struct{}{} // close signal to receive window + logs.Warn("receive mux connection close, finish", connection.connId) + }(connection) continue } } else if pack.Flag == common.MUX_CONN_CLOSE { - logs.Warn("rec mux conn close no id ", pack.Id) + logs.Warn("rec mux connection close no id ", pack.Id) continue } + common.MuxPack.Put(pack) } common.MuxPack.Put(pack) logs.Warn("read session put pack ", pack.Id) diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index 728dfa3..7c75c10 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -37,14 +37,17 @@ func TestNewMux(t *testing.T) { logs.Warn(err) continue } - var npcToServer common.ConnCopy - npcToServer.New(c2, c, 0) - go npcToServer.CopyConn() - var serverToNpc common.ConnCopy - serverToNpc.New(c, c2, 10000) - _, err = serverToNpc.CopyConn() - if err == nil { - logs.Warn("close npc") + go func() { + _, err = common.CopyBuffer(c2, c) + if err != nil { + logs.Warn("close npc by copy from nps", err) + c2.Close() + c.Close() + } + }() + _, err = common.CopyBuffer(c, c2) + if err != nil { + logs.Warn("close npc by copy from server", err) c2.Close() c.Close() } @@ -71,14 +74,18 @@ func TestNewMux(t *testing.T) { continue } logs.Warn("nps new conn success ", tmpCpnn.connId) - var userToNps common.ConnCopy - userToNps.New(tmpCpnn, conn, tmpCpnn.connId) - go userToNps.CopyConn() - var npsToUser common.ConnCopy - npsToUser.New(conn, tmpCpnn, tmpCpnn.connId+10000) - _, err = npsToUser.CopyConn() - if err == nil { - logs.Warn("close from out nps ", tmpCpnn.connId) + go func() { + _, err := common.CopyBuffer(tmpCpnn, conn) + if err != nil { + logs.Warn("close nps by copy from user", tmpCpnn.connId) + conn.Close() + tmpCpnn.Close() + } + }() + //time.Sleep(time.Second) + _, err = common.CopyBuffer(conn, tmpCpnn) + if err != nil { + logs.Warn("close nps by copy from npc ", tmpCpnn.connId) conn.Close() tmpCpnn.Close() } From 1c1be202b7e387c0cf74dd294640054ef5fa8596 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Wed, 11 Sep 2019 20:19:14 +0800 Subject: [PATCH 10/38] test for user close problem --- lib/mux/conn.go | 3 ++- lib/mux/mux.go | 20 ++++++++++++-------- lib/mux/mux_test.go | 28 +++++++++++++++++++--------- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 08b0db0..8f9f3ec 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -52,6 +52,7 @@ func NewConn(connId int32, mux *Mux) *conn { } func (s *conn) Read(buf []byte) (n int, err error) { + logs.Warn("starting conn read", s.connId) if s.isClose || buf == nil { return 0, errors.New("the conn has closed") } @@ -315,7 +316,7 @@ func (Self *window) allowRead() (closed bool) { } func (Self *window) Read(p []byte) (n int, err error) { - //logs.Warn("starting window read method len ", Self.len()) + logs.Warn("starting window read method len ", Self.len()) if Self.closeOp { return 0, io.EOF // Write method receive close signal, returns eof } diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 9310c64..5ad43c4 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -179,12 +179,14 @@ func (s *Mux) readSession() { switch pack.Flag { case common.MUX_NEW_CONN: //new connection logs.Warn("rec mux new connection", pack.Id) - conn := NewConn(pack.Id, s) - s.connMap.Set(pack.Id, conn) //it has been set before send ok - s.newConnCh <- conn - go conn.sendWindow.SetAllowSize(512) // set the initial receive window - s.sendInfo(common.MUX_NEW_CONN_OK, pack.Id, nil) - logs.Warn("send mux new connection ok", pack.Id) + connection := NewConn(pack.Id, s) + s.connMap.Set(pack.Id, connection) //it has been set before send ok + go func(connection *conn) { + connection.sendWindow.SetAllowSize(512) // set the initial receive window + }(connection) + s.newConnCh <- connection + s.sendInfo(common.MUX_NEW_CONN_OK, connection.connId, nil) + logs.Warn("send mux new connection ok", connection.connId) continue case common.MUX_PING_FLAG: //ping //logs.Warn("send mux ping return") @@ -202,7 +204,7 @@ func (s *Mux) readSession() { continue } connection.receiveWindow.WriteWg.Add(1) - logs.Warn("rec mux new msg ", pack.Id, string(pack.Content[0:15])) + logs.Warn("rec mux new msg ", connection.connId, string(pack.Content[0:15])) go func(connection *conn, content []byte) { // do not block read session _, err := connection.receiveWindow.Write(content) if err != nil { @@ -214,7 +216,7 @@ func (s *Mux) readSession() { connection.receiveWindow.WindowFull = true } s.sendInfo(common.MUX_MSG_SEND_OK, connection.connId, size) - logs.Warn("send mux new msg ok", pack.Id, size) + logs.Warn("send mux new msg ok", connection.connId, size) connection.receiveWindow.WriteWg.Done() }(connection, pack.Content) continue @@ -241,7 +243,9 @@ func (s *Mux) readSession() { s.connMap.Delete(pack.Id) connection.closeFlag = true go func(connection *conn) { + logs.Warn("receive mux connection close, wg waiting", connection.connId) connection.receiveWindow.WriteWg.Wait() + logs.Warn("receive mux connection close, wg waited", connection.connId) connection.receiveWindow.WriteEndOp <- struct{}{} // close signal to receive window logs.Warn("receive mux connection close, finish", connection.connId) }(connection) diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index 7c75c10..91e0fc6 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -6,6 +6,7 @@ import ( "net" "net/http" _ "net/http/pprof" + "sync" "testing" "time" ) @@ -37,20 +38,29 @@ func TestNewMux(t *testing.T) { logs.Warn(err) continue } + wg := sync.WaitGroup{} + wg.Add(1) go func() { _, err = common.CopyBuffer(c2, c) if err != nil { - logs.Warn("close npc by copy from nps", err) c2.Close() c.Close() + logs.Warn("close npc by copy from nps", err) } + wg.Done() }() - _, err = common.CopyBuffer(c, c2) - if err != nil { - logs.Warn("close npc by copy from server", err) - c2.Close() - c.Close() - } + wg.Add(1) + go func() { + _, err = common.CopyBuffer(c, c2) + if err != nil { + c2.Close() + c.Close() + logs.Warn("close npc by copy from server", err) + } + wg.Done() + }() + logs.Warn("npc wait") + wg.Wait() } }() @@ -77,17 +87,17 @@ func TestNewMux(t *testing.T) { go func() { _, err := common.CopyBuffer(tmpCpnn, conn) if err != nil { - logs.Warn("close nps by copy from user", tmpCpnn.connId) conn.Close() tmpCpnn.Close() + logs.Warn("close nps by copy from user", tmpCpnn.connId) } }() //time.Sleep(time.Second) _, err = common.CopyBuffer(conn, tmpCpnn) if err != nil { - logs.Warn("close nps by copy from npc ", tmpCpnn.connId) conn.Close() tmpCpnn.Close() + logs.Warn("close nps by copy from npc ", tmpCpnn.connId) } } }() From 6157b1a5281ceb4abbb1f597923413abfaf6bb43 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sun, 15 Sep 2019 15:02:10 +0800 Subject: [PATCH 11/38] fix block problems --- lib/mux/conn.go | 120 +++++++++++++++++++++++------------------------- lib/mux/mux.go | 44 +++++++++--------- 2 files changed, 80 insertions(+), 84 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 8f9f3ec..be73e5c 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -17,22 +17,15 @@ type conn struct { connStatusFailCh chan struct{} readTimeOut time.Time writeTimeOut time.Time - //readBuffer []byte - //startRead int //now read position - //endRead int //now end read - //readFlag bool - //readCh chan struct{} - //readQueue *sliceEntry - //stopWrite bool - connId int32 - isClose bool - //readWait bool - closeFlag bool // close conn flag - hasWrite int - receiveWindow *window - sendWindow *window - mux *Mux - once sync.Once + connId int32 + isClose bool + closeFlag bool // close conn flag + receiveWindow *window + sendWindow *window + readCh waitingCh + writeCh waitingCh + mux *Mux + once sync.Once } func NewConn(connId int32, mux *Mux) *conn { @@ -48,34 +41,32 @@ func NewConn(connId int32, mux *Mux) *conn { } c.receiveWindow.NewReceive() c.sendWindow.NewSend() + c.readCh.new() + c.writeCh.new() return c } func (s *conn) Read(buf []byte) (n int, err error) { - logs.Warn("starting conn read", s.connId) + //logs.Warn("starting conn read", s.connId) if s.isClose || buf == nil { return 0, errors.New("the conn has closed") } - nCh := make(chan int) - errCh := make(chan error) - defer close(nCh) - defer close(errCh) // waiting for takeout from receive window finish or timeout - go s.readWindow(buf, nCh, errCh) + go s.readWindow(buf, s.readCh.nCh, s.readCh.errCh) if t := s.readTimeOut.Sub(time.Now()); t > 0 { timer := time.NewTimer(t) defer timer.Stop() select { case <-timer.C: return 0, errors.New("read timeout") - case n = <-nCh: - err = <-errCh + case n = <-s.readCh.nCh: + err = <-s.readCh.errCh } } else { - n = <-nCh - err = <-errCh + n = <-s.readCh.nCh + err = <-s.readCh.errCh } - logs.Warn("read window finish conn read n err buf", n, err, string(buf[:15])) + //logs.Warn("read window finish conn read n err buf", n, err, string(buf[:15])) return } @@ -97,21 +88,19 @@ func (s *conn) readWindow(buf []byte, nCh chan int, errCh chan error) { } func (s *conn) Write(buf []byte) (n int, err error) { + //logs.Warn("write starting", s.connId) + //defer logs.Warn("write end ", s.connId) if s.isClose { return 0, errors.New("the conn has closed") } if s.closeFlag { - logs.Warn("conn close by write ", s.connId) + //logs.Warn("conn close by write ", s.connId) //s.Close() return 0, errors.New("io: write on closed conn") } - - nCh := make(chan int) - errCh := make(chan error) - defer close(nCh) - defer close(errCh) s.sendWindow.SetSendBuf(buf) // set the buf to send window - go s.write(nCh, errCh) + //logs.Warn("write set send buf success") + go s.write(s.writeCh.nCh, s.writeCh.errCh) // waiting for send to other side or timeout if t := s.writeTimeOut.Sub(time.Now()); t > 0 { timer := time.NewTimer(t) @@ -119,12 +108,12 @@ func (s *conn) Write(buf []byte) (n int, err error) { select { case <-timer.C: return 0, errors.New("write timeout") - case n = <-nCh: - err = <-errCh + case n = <-s.writeCh.nCh: + err = <-s.writeCh.errCh } } else { - n = <-nCh - err = <-errCh + n = <-s.writeCh.nCh + err = <-s.writeCh.errCh } return } @@ -160,7 +149,7 @@ func (s *conn) closeProcess() { s.isClose = true s.mux.connMap.Delete(s.connId) if !s.mux.IsClose { - logs.Warn("conn send close", s.connId) + //logs.Warn("conn send close", s.connId) // if server or user close the conn while reading, will get a io.EOF // and this Close method will be invoke, send this signal to close other side s.mux.sendInfo(common.MUX_CONN_CLOSE, s.connId, nil) @@ -198,7 +187,6 @@ type window struct { windowBuff []byte off uint16 readOp chan struct{} - readWait bool WindowFull bool usableReceiveWindow chan uint16 WriteWg sync.WaitGroup @@ -213,13 +201,13 @@ func (Self *window) NewReceive() { Self.windowBuff = common.WindowBuff.Get() Self.readOp = make(chan struct{}) Self.WriteEndOp = make(chan struct{}) - Self.closeOpCh = make(chan struct{}, 2) + Self.closeOpCh = make(chan struct{}, 3) } func (Self *window) NewSend() { // initial a window for send Self.usableReceiveWindow = make(chan uint16) - Self.closeOpCh = make(chan struct{}, 2) + Self.closeOpCh = make(chan struct{}, 3) } func (Self *window) SetSendBuf(buf []byte) { @@ -269,38 +257,32 @@ func (Self *window) grow(n int) { func (Self *window) Write(p []byte) (n int, err error) { if Self.closeOp { - logs.Warn("window write closed len p", len(p)) return 0, errors.New("conn.receiveWindow: write on closed window") } if len(p) > Self.Size() { return 0, errors.New("conn.receiveWindow: write too large") } - if Self.readWait { - defer Self.allowRead() - } - //logs.Warn("window write p string", len(p), string(p[:15])) Self.mutex.Lock() // slide the offset if len(p) > Self.cap()-Self.len() { // not enough space, need to allocate Self.fullSlide() - //logs.Warn("window write full slide len cap", Self.len(), Self.cap()) } else { // have enough space, re slice Self.liteSlide() - //logs.Warn("window write lite slide len cap", Self.len(), Self.cap()) } length := Self.len() // length before grow Self.grow(len(p)) // grow for copy n = copy(Self.windowBuff[length:], p) // must copy data before allow Read - //logs.Warn("window write copy n len cap buf", n, Self.len(), Self.cap(), string(Self.windowBuff[Self.len()-n:Self.len()+15-n])) + if length == 0 { + // allow continue read + defer Self.allowRead() + } Self.mutex.Unlock() return n, nil } func (Self *window) allowRead() (closed bool) { - //logs.Warn("length 0 read op") - Self.readWait = false if Self.closeOp { close(Self.readOp) return true @@ -310,25 +292,25 @@ func (Self *window) allowRead() (closed bool) { close(Self.readOp) return true case Self.readOp <- struct{}{}: - //logs.Warn("length 0 read op finish") return false } } func (Self *window) Read(p []byte) (n int, err error) { - logs.Warn("starting window read method len ", Self.len()) if Self.closeOp { return 0, io.EOF // Write method receive close signal, returns eof } - if Self.len() == 0 { + Self.mutex.Lock() + length := Self.len() // protect the length data, it invokes + // before Write lock and after Write unlock + Self.mutex.Unlock() + if length == 0 { // window is empty, waiting for Write method send a success readOp signal // or get timeout or close - Self.readWait = true ticker := time.NewTicker(2 * time.Minute) defer ticker.Stop() select { case _, ok := <-Self.readOp: - //logs.Warn("read window read op len cap", Self.len(), Self.cap()) if !ok { return 0, errors.New("conn.receiveWindow: window closed") } @@ -341,19 +323,17 @@ func (Self *window) Read(p []byte) (n int, err error) { return 0, io.EOF // receive close signal, returns eof } } - //logs.Warn("window read start len window buff", Self.len(), string(Self.windowBuff[Self.off:Self.off+15])) Self.mutex.Lock() n = copy(p, Self.windowBuff[Self.off:]) Self.off += uint16(n) p = p[:n] - //logs.Warn("window read finish n len p p", n, len(p), string(p[:15])) Self.mutex.Unlock() return } func (Self *window) WriteTo() (p []byte, err error) { if Self.closeOp { - logs.Warn("window write to closed") + //logs.Warn("window write to closed") return nil, errors.New("conn.writeWindow: window closed") } if Self.len() == 0 { @@ -373,6 +353,8 @@ waiting: } case <-ticker.C: return nil, errors.New("conn.writeWindow: write to time out") + case <-Self.closeOpCh: + return nil, errors.New("conn.writeWindow: window closed") } if windowSize == 0 { goto waiting // waiting for another usable window size @@ -380,10 +362,8 @@ waiting: Self.mutex.Lock() if windowSize > uint16(Self.len()) { // usable window size is bigger than window buff size, send the full buff - //logs.Warn("window size overflow windowSize len()", windowSize, Self.len()) windowSize = uint16(Self.len()) } - //logs.Warn("window buff off windowSize", Self.off, windowSize) p = Self.windowBuff[Self.off : windowSize+Self.off] Self.off += windowSize Self.mutex.Unlock() @@ -413,6 +393,22 @@ func (Self *window) CloseWindow() { Self.closeOp = true Self.closeOpCh <- struct{}{} Self.closeOpCh <- struct{}{} + Self.closeOpCh <- struct{}{} close(Self.closeOpCh) return } + +type waitingCh struct { + nCh chan int + errCh chan error +} + +func (Self *waitingCh) new() { + Self.nCh = make(chan int) + Self.errCh = make(chan error) +} + +func (Self *waitingCh) close() { + close(Self.nCh) + close(Self.errCh) +} diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 5ad43c4..dd3dee2 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -53,7 +53,7 @@ func (s *Mux) NewConn() (*conn, error) { //it must be set before send s.connMap.Set(conn.connId, conn) s.sendInfo(common.MUX_NEW_CONN, conn.connId, nil) - logs.Warn("send mux new conn ", conn.connId) + //logs.Warn("send mux new conn ", conn.connId) //set a timer timeout 30 second timer := time.NewTimer(time.Minute * 2) defer timer.Stop() @@ -85,7 +85,7 @@ func (s *Mux) sendInfo(flag uint8, id int32, data interface{}) { var err error if flag == common.MUX_NEW_MSG { if len(data.([]byte)) == 0 { - logs.Warn("send info content is nil") + //logs.Warn("send info content is nil") } } buf := common.BuffPool.Get() @@ -94,18 +94,18 @@ func (s *Mux) sendInfo(flag uint8, id int32, data interface{}) { defer common.MuxPack.Put(pack) err = pack.NewPac(flag, id, data) if err != nil { - logs.Warn("new pack err", err) + //logs.Warn("new pack err", err) common.BuffPool.Put(buf) return } err = pack.Pack(buf) if err != nil { - logs.Warn("pack err", err) + //logs.Warn("pack err", err) common.BuffPool.Put(buf) return } if pack.Flag == common.MUX_NEW_CONN { - logs.Warn("sendinfo mux new conn, insert to write queue", pack.Id) + //logs.Warn("sendinfo mux new conn, insert to write queue", pack.Id) } s.writeQueue <- buf //_, err = buf.WriteTo(s.conn) @@ -125,7 +125,7 @@ func (s *Mux) writeSession() { n, err := buf.WriteTo(s.conn) common.BuffPool.Put(buf) if err != nil || int(n) != l { - logs.Warn("close from write session fail ", err, n, l) + //logs.Warn("close from write session fail ", err, n, l) s.Close() break } @@ -173,12 +173,12 @@ func (s *Mux) readSession() { } } if pack.Flag == common.MUX_NEW_CONN { - logs.Warn("unpack mux new connection", pack.Id) + //logs.Warn("unpack mux new connection", pack.Id) } s.pingOk = 0 switch pack.Flag { case common.MUX_NEW_CONN: //new connection - logs.Warn("rec mux new connection", pack.Id) + //logs.Warn("rec mux new connection", pack.Id) connection := NewConn(pack.Id, s) s.connMap.Set(pack.Id, connection) //it has been set before send ok go func(connection *conn) { @@ -186,7 +186,7 @@ func (s *Mux) readSession() { }(connection) s.newConnCh <- connection s.sendInfo(common.MUX_NEW_CONN_OK, connection.connId, nil) - logs.Warn("send mux new connection ok", connection.connId) + //logs.Warn("send mux new connection ok", connection.connId) continue case common.MUX_PING_FLAG: //ping //logs.Warn("send mux ping return") @@ -204,61 +204,61 @@ func (s *Mux) readSession() { continue } connection.receiveWindow.WriteWg.Add(1) - logs.Warn("rec mux new msg ", connection.connId, string(pack.Content[0:15])) + //logs.Warn("rec mux new msg ", connection.connId, string(pack.Content[0:15])) go func(connection *conn, content []byte) { // do not block read session _, err := connection.receiveWindow.Write(content) if err != nil { logs.Warn("mux new msg err close", err) - s.Close() + connection.Close() } size := connection.receiveWindow.Size() if size == 0 { connection.receiveWindow.WindowFull = true } s.sendInfo(common.MUX_MSG_SEND_OK, connection.connId, size) - logs.Warn("send mux new msg ok", connection.connId, size) + //logs.Warn("send mux new msg ok", connection.connId, size) connection.receiveWindow.WriteWg.Done() }(connection, pack.Content) continue case common.MUX_NEW_CONN_OK: //connection ok - logs.Warn("rec mux new connection ok ", pack.Id) + //logs.Warn("rec mux new connection ok ", pack.Id) connection.connStatusOkCh <- struct{}{} go connection.sendWindow.SetAllowSize(512) // set the initial receive window both side continue case common.MUX_NEW_CONN_Fail: - logs.Warn("rec mux new connection fail", pack.Id) + //logs.Warn("rec mux new connection fail", pack.Id) connection.connStatusFailCh <- struct{}{} continue case common.MUX_MSG_SEND_OK: if connection.isClose { - logs.Warn("rec mux msg send ok id window closed!", pack.Id, pack.Window) + //logs.Warn("rec mux msg send ok id window closed!", pack.Id, pack.Window) continue } - logs.Warn("rec mux msg send ok id window", pack.Id, pack.Window) + //logs.Warn("rec mux msg send ok id window", pack.Id, pack.Window) go connection.sendWindow.SetAllowSize(pack.Window) continue case common.MUX_CONN_CLOSE: //close the connection - logs.Warn("rec mux connection close", pack.Id) + //logs.Warn("rec mux connection close", pack.Id) s.connMap.Delete(pack.Id) connection.closeFlag = true go func(connection *conn) { - logs.Warn("receive mux connection close, wg waiting", connection.connId) + //logs.Warn("receive mux connection close, wg waiting", connection.connId) connection.receiveWindow.WriteWg.Wait() - logs.Warn("receive mux connection close, wg waited", connection.connId) + //logs.Warn("receive mux connection close, wg waited", connection.connId) connection.receiveWindow.WriteEndOp <- struct{}{} // close signal to receive window - logs.Warn("receive mux connection close, finish", connection.connId) + //logs.Warn("receive mux connection close, finish", connection.connId) }(connection) continue } } else if pack.Flag == common.MUX_CONN_CLOSE { - logs.Warn("rec mux connection close no id ", pack.Id) + //logs.Warn("rec mux connection close no id ", pack.Id) continue } common.MuxPack.Put(pack) } common.MuxPack.Put(pack) - logs.Warn("read session put pack ", pack.Id) + //logs.Warn("read session put pack ", pack.Id) s.Close() }() select { From 1bf4cf23478b43ee151a0408411cfb01211755bb Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Tue, 17 Sep 2019 19:05:04 +0800 Subject: [PATCH 12/38] fix conn write block, add priority queue support --- lib/mux/conn.go | 15 ++++- lib/mux/mux.go | 103 +++++++++++++++++++++------------- lib/mux/queue.go | 143 ++++++++++++++++++++++++++--------------------- 3 files changed, 156 insertions(+), 105 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index be73e5c..81f876f 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -187,6 +187,7 @@ type window struct { windowBuff []byte off uint16 readOp chan struct{} + readWait bool WindowFull bool usableReceiveWindow chan uint16 WriteWg sync.WaitGroup @@ -274,7 +275,11 @@ func (Self *window) Write(p []byte) (n int, err error) { length := Self.len() // length before grow Self.grow(len(p)) // grow for copy n = copy(Self.windowBuff[length:], p) // must copy data before allow Read - if length == 0 { + if Self.readWait { + // if there condition is length == 0 and + // Read method just take away all the windowBuff, + // this method will block until windowBuff is empty again + // allow continue read defer Self.allowRead() } @@ -287,6 +292,9 @@ func (Self *window) allowRead() (closed bool) { close(Self.readOp) return true } + Self.mutex.Lock() + Self.readWait = false + Self.mutex.Unlock() select { case <-Self.closeOpCh: close(Self.readOp) @@ -303,10 +311,11 @@ func (Self *window) Read(p []byte) (n int, err error) { Self.mutex.Lock() length := Self.len() // protect the length data, it invokes // before Write lock and after Write unlock - Self.mutex.Unlock() if length == 0 { // window is empty, waiting for Write method send a success readOp signal // or get timeout or close + Self.readWait = true + Self.mutex.Unlock() ticker := time.NewTicker(2 * time.Minute) defer ticker.Stop() select { @@ -322,6 +331,8 @@ func (Self *window) Read(p []byte) (n int, err error) { close(Self.readOp) return 0, io.EOF // receive close signal, returns eof } + } else { + Self.mutex.Unlock() } Self.mutex.Lock() n = copy(p, Self.windowBuff[Self.off:]) diff --git a/lib/mux/mux.go b/lib/mux/mux.go index dd3dee2..f4bc6fa 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -22,21 +22,23 @@ type Mux struct { IsClose bool pingOk int connType string - writeQueue chan *bytes.Buffer + writeQueue Queue + bufCh chan *bytes.Buffer sync.Mutex } func NewMux(c net.Conn, connType string) *Mux { m := &Mux{ - conn: c, - connMap: NewConnMap(), - id: 0, - closeChan: make(chan struct{}), - newConnCh: make(chan *conn), - IsClose: false, - connType: connType, - writeQueue: make(chan *bytes.Buffer, 20), + conn: c, + connMap: NewConnMap(), + id: 0, + closeChan: make(chan struct{}), + newConnCh: make(chan *conn), + IsClose: false, + connType: connType, + bufCh: make(chan *bytes.Buffer), } + m.writeQueue.New() //read session by flag go m.readSession() //ping @@ -88,26 +90,27 @@ func (s *Mux) sendInfo(flag uint8, id int32, data interface{}) { //logs.Warn("send info content is nil") } } - buf := common.BuffPool.Get() + //buf := common.BuffPool.Get() //defer pool.BuffPool.Put(buf) pack := common.MuxPack.Get() - defer common.MuxPack.Put(pack) + err = pack.NewPac(flag, id, data) if err != nil { //logs.Warn("new pack err", err) - common.BuffPool.Put(buf) + common.MuxPack.Put(pack) return } - err = pack.Pack(buf) - if err != nil { - //logs.Warn("pack err", err) - common.BuffPool.Put(buf) - return - } - if pack.Flag == common.MUX_NEW_CONN { - //logs.Warn("sendinfo mux new conn, insert to write queue", pack.Id) - } - s.writeQueue <- buf + s.writeQueue.Push(pack) + //err = pack.Pack(buf) + //if err != nil { + // //logs.Warn("pack err", err) + // common.BuffPool.Put(buf) + // return + //} + //if pack.Flag == common.MUX_NEW_CONN { + // //logs.Warn("sendinfo mux new conn, insert to write queue", pack.Id) + //} + //s.writeQueue <- buf //_, err = buf.WriteTo(s.conn) //if err != nil { // s.Close() @@ -118,20 +121,47 @@ func (s *Mux) sendInfo(flag uint8, id int32, data interface{}) { } func (s *Mux) writeSession() { - go func() { - for { - buf := <-s.writeQueue - l := buf.Len() - n, err := buf.WriteTo(s.conn) - common.BuffPool.Put(buf) + go s.packBuf() + go s.writeBuf() + <-s.closeChan +} + +func (s *Mux) packBuf() { + for { + pack := s.writeQueue.Pop() + buffer := common.BuffPool.Get() + err := pack.Pack(buffer) + common.MuxPack.Put(pack) + if err != nil { + logs.Warn("pack err", err) + common.BuffPool.Put(buffer) + break + } + select { + case s.bufCh <- buffer: + case <-s.closeChan: + break + } + + } +} + +func (s *Mux) writeBuf() { + for { + select { + case buffer := <-s.bufCh: + l := buffer.Len() + n, err := buffer.WriteTo(s.conn) + common.BuffPool.Put(buffer) if err != nil || int(n) != l { - //logs.Warn("close from write session fail ", err, n, l) + logs.Warn("close from write session fail ", err, n, l) s.Close() break } + case <-s.closeChan: + break } - }() - <-s.closeChan + } } func (s *Mux) ping() { @@ -273,14 +303,11 @@ func (s *Mux) Close() error { } s.IsClose = true s.connMap.Close() - select { - case s.closeChan <- struct{}{}: - } - select { - case s.closeChan <- struct{}{}: - } s.closeChan <- struct{}{} - close(s.writeQueue) + s.closeChan <- struct{}{} + s.closeChan <- struct{}{} + s.closeChan <- struct{}{} + s.closeChan <- struct{}{} close(s.newConnCh) return s.conn.Close() } diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 4388fb6..081b2c9 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -1,82 +1,95 @@ package mux import ( - "errors" + "container/list" "github.com/cnlh/nps/lib/common" "sync" ) -type Element *bufNode - -type bufNode struct { - val []byte //buf value - l int //length +type Queue struct { + list *list.List + readOp chan struct{} + cleanOp chan struct{} + popWait bool + mutex sync.Mutex } -func NewBufNode(buf []byte, l int) *bufNode { - return &bufNode{ - val: buf, - l: l, +func (Self *Queue) New() { + Self.list = list.New() + Self.readOp = make(chan struct{}) + Self.cleanOp = make(chan struct{}, 2) +} + +func (Self *Queue) Push(packager *common.MuxPackager) { + Self.mutex.Lock() + if Self.popWait { + defer Self.allowPop() } -} - -type Queue interface { - Push(e Element) //向队列中添加元素 - Pop() Element //移除队列中最前面的元素 - Clear() bool //清空队列 - Size() int //获取队列的元素个数 - IsEmpty() bool //判断队列是否是空 -} - -type sliceEntry struct { - element []Element - sync.Mutex -} - -func NewQueue() *sliceEntry { - return &sliceEntry{} -} - -//向队列中添加元素 -func (entry *sliceEntry) Push(e Element) { - entry.Lock() - defer entry.Unlock() - entry.element = append(entry.element, e) -} - -//移除队列中最前面的额元素 -func (entry *sliceEntry) Pop() (Element, error) { - if entry.IsEmpty() { - return nil, errors.New("queue is empty!") + if packager.Flag == common.MUX_CONN_CLOSE { + Self.insert(packager) // the close package may need priority, + // prevent wait too long to close + } else { + Self.list.PushBack(packager) } - entry.Lock() - defer entry.Unlock() - firstElement := entry.element[0] - entry.element = entry.element[1:] - return firstElement, nil + Self.mutex.Unlock() + return } -func (entry *sliceEntry) Clear() bool { - entry.Lock() - defer entry.Unlock() - if entry.IsEmpty() { +func (Self *Queue) allowPop() (closed bool) { + Self.mutex.Lock() + Self.popWait = false + Self.mutex.Unlock() + select { + case Self.readOp <- struct{}{}: return false - } - for i := 0; i < entry.Size(); i++ { - common.CopyBuff.Put(entry.element[i].val) - entry.element[i] = nil - } - entry.element = nil - return true -} - -func (entry *sliceEntry) Size() int { - return len(entry.element) -} - -func (entry *sliceEntry) IsEmpty() bool { - if len(entry.element) == 0 { + case <-Self.cleanOp: return true } - return false +} + +func (Self *Queue) insert(packager *common.MuxPackager) { + element := Self.list.Back() + for { + if element == nil { // Queue dose not have any of msg package with this close package id + Self.list.PushFront(packager) // insert close package to first + break + } + if element.Value.(*common.MuxPackager).Flag == common.MUX_NEW_MSG && + element.Value.(*common.MuxPackager).Id == packager.Id { + Self.list.InsertAfter(packager, element) // Queue has some msg package + // with this close package id, insert close package after last msg package + break + } + element = element.Prev() + } +} + +func (Self *Queue) Pop() (packager *common.MuxPackager) { + Self.mutex.Lock() + element := Self.list.Front() + if element != nil { + packager = element.Value.(*common.MuxPackager) + Self.list.Remove(element) + Self.mutex.Unlock() + return + } + Self.popWait = true // Queue is empty, notice Push method + Self.mutex.Unlock() + select { + case <-Self.readOp: + return Self.Pop() + case <-Self.cleanOp: + return nil + } +} + +func (Self *Queue) Len() (n int) { + n = Self.list.Len() + return +} + +func (Self *Queue) Clean() { + Self.cleanOp <- struct{}{} + Self.cleanOp <- struct{}{} + close(Self.cleanOp) } From f0201c103926ad7c4054fcfac510df5eecd70c65 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sun, 22 Sep 2019 22:08:51 +0800 Subject: [PATCH 13/38] change slide window --- lib/mux/conn.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 81f876f..b1141ee 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -66,7 +66,7 @@ func (s *conn) Read(buf []byte) (n int, err error) { n = <-s.readCh.nCh err = <-s.readCh.errCh } - //logs.Warn("read window finish conn read n err buf", n, err, string(buf[:15])) + //logs.Warn("read window finish conn read n err buf", n, err, string(buf[:15]), s.connId) return } @@ -115,6 +115,7 @@ func (s *conn) Write(buf []byte) (n int, err error) { n = <-s.writeCh.nCh err = <-s.writeCh.errCh } + //logs.Warn("write window finish n err buf id", n, err, string(buf[:15]), s.connId) return } func (s *conn) write(nCh chan int, errCh chan error) { @@ -334,11 +335,22 @@ func (Self *window) Read(p []byte) (n int, err error) { } else { Self.mutex.Unlock() } - Self.mutex.Lock() - n = copy(p, Self.windowBuff[Self.off:]) - Self.off += uint16(n) + minCopy := 512 + for { + Self.mutex.Lock() + if len(p) == n || Self.len() == 0 { + Self.mutex.Unlock() + break + } + if n+minCopy > len(p) { + minCopy = len(p) - n + } + i := copy(p[n:n+minCopy], Self.windowBuff[Self.off:]) + Self.off += uint16(i) + n += i + Self.mutex.Unlock() + } p = p[:n] - Self.mutex.Unlock() return } From 847f0ce1d495b642b531f61816c54b2c1bbded6a Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Mon, 23 Sep 2019 23:09:58 +0800 Subject: [PATCH 14/38] fix window write loss --- lib/mux/conn.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index b1141ee..3f824cc 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -223,10 +223,10 @@ func (Self *window) SetSendBuf(buf []byte) { func (Self *window) fullSlide() { // slide by allocate newBuf := common.WindowBuff.Get() - copy(newBuf[0:Self.len()], Self.windowBuff[Self.off:]) - Self.off = 0 + Self.liteSlide() + n := copy(newBuf[:Self.len()], Self.windowBuff) common.WindowBuff.Put(Self.windowBuff) - Self.windowBuff = newBuf + Self.windowBuff = newBuf[:n] return } From a61ff2d2008a7fbb8599cf1a6ab6b0a20c6548a4 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Tue, 24 Sep 2019 22:29:31 +0800 Subject: [PATCH 15/38] merge --- go.mod | 7 ++-- go.sum | 8 +++++ lib/common/netpackager.go | 3 -- lib/mux/conn.go | 1 - lib/mux/mux.go | 3 +- lib/mux/mux_test.go | 74 ++++++++++++++++++++------------------- 6 files changed, 51 insertions(+), 45 deletions(-) diff --git a/go.mod b/go.mod index 1f6b753..8a19eaf 100644 --- a/go.mod +++ b/go.mod @@ -10,18 +10,17 @@ require ( github.com/go-ole/go-ole v1.2.4 // indirect github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db github.com/klauspost/cpuid v1.2.1 // indirect - github.com/klauspost/reedsolomon v1.9.2 + github.com/klauspost/reedsolomon v1.9.2 // indirect github.com/onsi/gomega v1.5.0 // indirect github.com/pkg/errors v0.8.0 github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 // indirect github.com/shirou/gopsutil v2.18.12+incompatible github.com/stretchr/testify v1.3.0 // indirect github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161 // indirect - github.com/templexxx/xor v0.0.0-20181023030647-4e92f724b73b - github.com/tjfoc/gmsm v1.0.1 + github.com/templexxx/xor v0.0.0-20181023030647-4e92f724b73b // indirect + github.com/tjfoc/gmsm v1.0.1 // indirect github.com/xtaci/kcp-go v5.4.4+incompatible github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae // indirect - golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa // indirect ) diff --git a/go.sum b/go.sum index 29de132..f3a17f4 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OwnLocal/goes v1.0.0/go.mod h1:8rIFjBGTue3lCU0wplczcUgt9Gxgrkkrw7etMIcn8TM= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/astaxie/beego v1.12.0 h1:MRhVoeeye5N+Flul5PoVfD9CslfdoH+xqC/xvSQ5u2Y= github.com/astaxie/beego v1.12.0/go.mod h1:fysx+LZNZKnvh4GED/xND7jWtjCR6HzydR2Hh2Im57o= @@ -15,14 +16,17 @@ github.com/couchbase/go-couchbase v0.0.0-20181122212707-3e9b6e1258bb/go.mod h1:T github.com/couchbase/gomemcached v0.0.0-20181122193126-5125a94a666c/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c= github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs= github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGiisLwp9rITslkFNpZD5rz43tf41QFkTWY= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elazarl/go-bindata-assetfs v1.0.0 h1:G/bYguwHIzWq9ZoyUQqrjTmJbbYn3j3CKKpKinvZLFk= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/exfly/beego v1.12.0-export-init h1:VQNYKdXhAwZGUaFmQv8Aj921O3rQJZRIF8xeGrhsjrI= github.com/exfly/beego v1.12.0-export-init/go.mod h1:fysx+LZNZKnvh4GED/xND7jWtjCR6HzydR2Hh2Im57o= github.com/exfly/beego v1.12.0 h1:OXwIwngaAx35Mga+jLiZmArusBxj8/H0jYXzGDAdwOg= github.com/exfly/beego v1.12.0/go.mod h1:fysx+LZNZKnvh4GED/xND7jWtjCR6HzydR2Hh2Im57o= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -43,6 +47,7 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 h1:X+yvsM2yrEktyI+b2qND5gpH8YhURn0k8OCaeRnkINo= github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644/go.mod h1:nkxAfR/5quYxwPZhyDxgasBMnRtBZd0FCEpawpjMUFg= @@ -53,6 +58,7 @@ github.com/siddontang/ledisdb v0.0.0-20181029004158-becf5f38d373/go.mod h1:mF1Dp github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d/go.mod h1:AMEsy7v5z92TR1JKMkLLoaOQk++LVnOKL3ScbJ8GNGA= github.com/ssdb/gossdb v0.0.0-20180723034631-88f6b59b84ec/go.mod h1:QBvMkMya+gXctz3kmljlUCu/yB3GZ6oee+dUozsezQE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161 h1:89CEmDvlq/F7SJEOqkIdNDGJXrQIhuIx9D2DBXjavSU= @@ -64,6 +70,7 @@ github.com/tjfoc/gmsm v1.0.1/go.mod h1:XxO4hdhhrzAd+G4CjDqaOkd0hUzmtPR/d3EiBBMn/ github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc= github.com/xtaci/kcp-go v5.4.4+incompatible h1:QIJ0a0Q0N1G20yLHL2+fpdzyy2v/Cb3PI+xiwx/KK9c= github.com/xtaci/kcp-go v5.4.4+incompatible/go.mod h1:bN6vIwHQbfHaHtFpEssmWsN45a+AZwO7eyRCmEIbtvE= +github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae h1:J0GxkO96kL4WF+AIT3M4mfUVinOCPgf2uUWYFUzN0sM= github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE= golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85 h1:et7+NAX3lLIk5qUCTA9QelBjGE/NkhzYw/mhnr0s7nI= golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -75,6 +82,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa h1:KIDDMLT1O0Nr7TSxp8xM5tJcdn8tgyAONntO829og1M= golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/lib/common/netpackager.go b/lib/common/netpackager.go index f2ddaec..1129940 100644 --- a/lib/common/netpackager.go +++ b/lib/common/netpackager.go @@ -5,7 +5,6 @@ import ( "encoding/binary" "encoding/json" "errors" - "github.com/cnlh/nps/vender/github.com/astaxie/beego/logs" "io" "strings" ) @@ -50,7 +49,6 @@ func (Self *BasePackager) appendByte(data []byte) (err error) { copy(Self.Content[m:n], data) return nil } else { - logs.Warn(len(data), len(Self.Content), cap(Self.Content)) return errors.New("pack content too large") } } @@ -74,7 +72,6 @@ func (Self *BasePackager) UnPack(reader io.Reader) (err error) { return } if int(Self.Length) > cap(Self.Content) { - logs.Warn("unpack", cap(Self.Content)) err = errors.New("unpack err, content length too large") } Self.Content = Self.Content[:int(Self.Length)] diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 23ba78d..bf9e0d6 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -70,7 +70,6 @@ func (s *conn) Read(buf []byte) (n int, err error) { func (s *conn) readWindow(buf []byte, nCh chan int, errCh chan error) { n, err := s.receiveWindow.Read(buf) - //logs.Warn("readwindow goroutine status n err buf", n, err, string(buf[:15])) if s.receiveWindow.WindowFull { if s.receiveWindow.Size() > 0 { // window.Read may be invoked before window.Write, and WindowFull flag change to true diff --git a/lib/mux/mux.go b/lib/mux/mux.go index f955233..a662ad0 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "time" + "github.com/astaxie/beego/logs" "github.com/cnlh/nps/lib/common" ) @@ -173,7 +174,7 @@ func (s *Mux) readSession() { s.pingOk = 0 switch pack.Flag { case common.MUX_NEW_CONN: //new connection - connection := NewConn(pack.Id, s) + connection := NewConn(pack.Id, s) s.connMap.Set(pack.Id, connection) //it has been set before send ok go func(connection *conn) { connection.sendWindow.SetAllowSize(512) // set the initial receive window diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index 5f5c990..c7b10e0 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -1,7 +1,6 @@ package mux import ( - "log" "net" "net/http" _ "net/http/pprof" @@ -11,7 +10,6 @@ import ( "github.com/astaxie/beego/logs" "github.com/cnlh/nps/lib/common" - "github.com/cnlh/nps/lib/pool" ) var conn1 net.Conn @@ -41,29 +39,31 @@ func TestNewMux(t *testing.T) { logs.Warn(err) continue } - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - _, err = common.CopyBuffer(c2, c) - if err != nil { - c2.Close() - c.Close() - logs.Warn("close npc by copy from nps", err) - } - wg.Done() - }() - wg.Add(1) - go func() { - _, err = common.CopyBuffer(c, c2) - if err != nil { - c2.Close() - c.Close() - logs.Warn("close npc by copy from server", err) - } - wg.Done() - }() - logs.Warn("npc wait") - wg.Wait() + go func(c2 net.Conn, c net.Conn) { + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + _, err = common.CopyBuffer(c2, c) + if err != nil { + c2.Close() + c.Close() + logs.Warn("close npc by copy from nps", err) + } + wg.Done() + }() + wg.Add(1) + go func() { + _, err = common.CopyBuffer(c, c2) + if err != nil { + c2.Close() + c.Close() + logs.Warn("close npc by copy from server", err) + } + wg.Done() + }() + logs.Warn("npc wait") + wg.Wait() + }(c2, c) } }() @@ -87,21 +87,23 @@ func TestNewMux(t *testing.T) { continue } logs.Warn("nps new conn success ", tmpCpnn.connId) - go func() { - _, err := common.CopyBuffer(tmpCpnn, conn) + go func(tmpCpnn net.Conn, conn net.Conn) { + go func() { + _, err := common.CopyBuffer(tmpCpnn, conn) + if err != nil { + conn.Close() + tmpCpnn.Close() + logs.Warn("close nps by copy from user") + } + }() + //time.Sleep(time.Second) + _, err = common.CopyBuffer(conn, tmpCpnn) if err != nil { conn.Close() tmpCpnn.Close() - logs.Warn("close nps by copy from user", tmpCpnn.connId) + logs.Warn("close nps by copy from npc ") } - }() - //time.Sleep(time.Second) - _, err = common.CopyBuffer(conn, tmpCpnn) - if err != nil { - conn.Close() - tmpCpnn.Close() - logs.Warn("close nps by copy from npc ", tmpCpnn.connId) - } + }(tmpCpnn, conn) } }() From 8bcf5313f4d59175c835cb5dfe84d29dbe82c01d Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Wed, 25 Sep 2019 00:13:34 +0800 Subject: [PATCH 16/38] merge --- lib/conn/conn.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/conn/conn.go b/lib/conn/conn.go index 30e4cd6..7946c0d 100755 --- a/lib/conn/conn.go +++ b/lib/conn/conn.go @@ -6,13 +6,6 @@ import ( "encoding/binary" "encoding/json" "errors" - "github.com/cnlh/nps/lib/common" - "github.com/cnlh/nps/lib/crypt" - "github.com/cnlh/nps/lib/file" - "github.com/cnlh/nps/lib/mux" - "github.com/cnlh/nps/lib/pool" - "github.com/cnlh/nps/lib/rate" - "github.com/cnlh/nps/vender/github.com/xtaci/kcp" "io" "net" "net/http" From b3ed822c72c9ec31f873151edd4d16333046651a Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Mon, 7 Oct 2019 23:04:54 +0800 Subject: [PATCH 17/38] change slide window design --- lib/common/const.go | 4 +- lib/common/netpackager.go | 37 +- lib/mux/conn.go | 703 ++++++++++++++++++++++---------------- lib/mux/mux.go | 118 ++++--- lib/mux/queue.go | 148 ++++++-- 5 files changed, 623 insertions(+), 387 deletions(-) diff --git a/lib/common/const.go b/lib/common/const.go index d77f16b..95364a2 100644 --- a/lib/common/const.go +++ b/lib/common/const.go @@ -42,9 +42,11 @@ const ( MUX_NEW_CONN_OK MUX_NEW_CONN_Fail MUX_NEW_MSG + MUX_NEW_MSG_PART MUX_MSG_SEND_OK MUX_NEW_CONN MUX_CONN_CLOSE MUX_PING_RETURN - MUX_PING int32 = -1 + MUX_PING int32 = -1 + MAXIMUM_SEGMENT_SIZE = 4096 - 16 - 32 - 32 - 8 ) diff --git a/lib/common/netpackager.go b/lib/common/netpackager.go index 1129940..ec2cb69 100644 --- a/lib/common/netpackager.go +++ b/lib/common/netpackager.go @@ -15,7 +15,7 @@ type NetPackager interface { } type BasePackager struct { - Length uint32 + Length uint16 Content []byte } @@ -101,7 +101,7 @@ func (Self *BasePackager) Unmarshal(content interface{}) (err error) { } func (Self *BasePackager) setLength() { - Self.Length = uint32(len(Self.Content)) + Self.Length = uint16(len(Self.Content)) return } @@ -147,25 +147,32 @@ func (Self *ConnPackager) UnPack(reader io.Reader) (err error) { } type MuxPackager struct { - Flag uint8 - Id int32 - Window uint16 + Flag uint8 + Id int32 + Window uint32 + ReadLength uint32 BasePackager } func (Self *MuxPackager) NewPac(flag uint8, id int32, content ...interface{}) (err error) { Self.Flag = flag Self.Id = id - if flag == MUX_NEW_MSG { + if flag == MUX_NEW_MSG || flag == MUX_NEW_MSG_PART || flag == MUX_PING_FLAG { err = Self.BasePackager.NewPac(content...) } if flag == MUX_MSG_SEND_OK { // MUX_MSG_SEND_OK only allows one data switch content[0].(type) { case int: - Self.Window = uint16(content[0].(int)) - case uint16: - Self.Window = content[0].(uint16) + Self.Window = uint32(content[0].(int)) + case uint32: + Self.Window = content[0].(uint32) + } + switch content[1].(type) { + case int: + Self.ReadLength = uint32(content[1].(int)) + case uint32: + Self.ReadLength = content[1].(uint32) } } return @@ -180,11 +187,15 @@ func (Self *MuxPackager) Pack(writer io.Writer) (err error) { if err != nil { return } - if Self.Flag == MUX_NEW_MSG { + if Self.Flag == MUX_NEW_MSG || Self.Flag == MUX_NEW_MSG_PART || Self.Flag == MUX_PING_FLAG { err = Self.BasePackager.Pack(writer) } if Self.Flag == MUX_MSG_SEND_OK { err = binary.Write(writer, binary.LittleEndian, Self.Window) + if err != nil { + return + } + err = binary.Write(writer, binary.LittleEndian, Self.ReadLength) } return } @@ -199,11 +210,15 @@ func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { if err != nil { return } - if Self.Flag == MUX_NEW_MSG { + if Self.Flag == MUX_NEW_MSG || Self.Flag == MUX_NEW_MSG_PART || Self.Flag == MUX_PING_FLAG { err = Self.BasePackager.UnPack(reader) } if Self.Flag == MUX_MSG_SEND_OK { err = binary.Read(reader, binary.LittleEndian, &Self.Window) + if err != nil { + return + } + err = binary.Read(reader, binary.LittleEndian, &Self.ReadLength) } return } diff --git a/lib/mux/conn.go b/lib/mux/conn.go index bf9e0d6..f4d5396 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -2,7 +2,9 @@ package mux import ( "errors" + "github.com/astaxie/beego/logs" "io" + "math" "net" "sync" "time" @@ -15,16 +17,11 @@ type conn struct { getStatusCh chan struct{} connStatusOkCh chan struct{} connStatusFailCh chan struct{} - readTimeOut time.Time - writeTimeOut time.Time connId int32 isClose bool closeFlag bool // close conn flag - receiveWindow *window - sendWindow *window - readCh waitingCh - writeCh waitingCh - mux *Mux + receiveWindow *ReceiveWindow + sendWindow *SendWindow once sync.Once } @@ -34,15 +31,12 @@ func NewConn(connId int32, mux *Mux) *conn { connStatusOkCh: make(chan struct{}), connStatusFailCh: make(chan struct{}), connId: connId, - receiveWindow: new(window), - sendWindow: new(window), - mux: mux, + receiveWindow: new(ReceiveWindow), + sendWindow: new(SendWindow), once: sync.Once{}, } - c.receiveWindow.NewReceive() - c.sendWindow.NewSend() - c.readCh.new() - c.writeCh.new() + c.receiveWindow.New(mux) + c.sendWindow.New(mux) return c } @@ -50,39 +44,14 @@ func (s *conn) Read(buf []byte) (n int, err error) { if s.isClose || buf == nil { return 0, errors.New("the conn has closed") } + if len(buf) == 0 { + return 0, nil + } // waiting for takeout from receive window finish or timeout - go s.readWindow(buf, s.readCh.nCh, s.readCh.errCh) - if t := s.readTimeOut.Sub(time.Now()); t > 0 { - timer := time.NewTimer(t) - defer timer.Stop() - select { - case <-timer.C: - return 0, errors.New("read timeout") - case n = <-s.readCh.nCh: - err = <-s.readCh.errCh - } - } else { - n = <-s.readCh.nCh - err = <-s.readCh.errCh - } + n, err = s.receiveWindow.Read(buf, s.connId) return } -func (s *conn) readWindow(buf []byte, nCh chan int, errCh chan error) { - n, err := s.receiveWindow.Read(buf) - if s.receiveWindow.WindowFull { - if s.receiveWindow.Size() > 0 { - // window.Read may be invoked before window.Write, and WindowFull flag change to true - // so make sure that receiveWindow is free some space - s.receiveWindow.WindowFull = false - s.mux.sendInfo(common.MUX_MSG_SEND_OK, s.connId, s.receiveWindow.Size()) - // acknowledge other side, have empty some receive window space - } - } - nCh <- n - errCh <- err -} - func (s *conn) Write(buf []byte) (n int, err error) { if s.isClose { return 0, errors.New("the conn has closed") @@ -91,45 +60,13 @@ func (s *conn) Write(buf []byte) (n int, err error) { //s.Close() return 0, errors.New("io: write on closed conn") } - s.sendWindow.SetSendBuf(buf) // set the buf to send window - go s.write(s.writeCh.nCh, s.writeCh.errCh) - // waiting for send to other side or timeout - if t := s.writeTimeOut.Sub(time.Now()); t > 0 { - timer := time.NewTimer(t) - defer timer.Stop() - select { - case <-timer.C: - return 0, errors.New("write timeout") - case n = <-s.writeCh.nCh: - err = <-s.writeCh.errCh - } - } else { - n = <-s.writeCh.nCh - err = <-s.writeCh.errCh + if len(buf) == 0 { + return 0, nil } + //logs.Warn("write buf", len(buf)) + n, err = s.sendWindow.WriteFull(buf, s.connId) return } -func (s *conn) write(nCh chan int, errCh chan error) { - var n int - var err error - for { - buf, err := s.sendWindow.WriteTo() - // get the usable window size buf from send window - if buf == nil && err == io.EOF { - // send window is drain, break the loop - err = nil - break - } - if err != nil { - break - } - n += len(buf) - s.mux.sendInfo(common.MUX_NEW_MSG, s.connId, buf) - // send to other side, not send nil data to other side - } - nCh <- n - errCh <- err -} func (s *conn) Close() (err error) { s.once.Do(s.closeProcess) @@ -138,11 +75,11 @@ func (s *conn) Close() (err error) { func (s *conn) closeProcess() { s.isClose = true - s.mux.connMap.Delete(s.connId) - if !s.mux.IsClose { + s.receiveWindow.mux.connMap.Delete(s.connId) + if !s.receiveWindow.mux.IsClose { // if server or user close the conn while reading, will get a io.EOF // and this Close method will be invoke, send this signal to close other side - s.mux.sendInfo(common.MUX_CONN_CLOSE, s.connId, nil) + s.receiveWindow.mux.sendInfo(common.MUX_CONN_CLOSE, s.connId, nil) } s.sendWindow.CloseWindow() s.receiveWindow.CloseWindow() @@ -150,276 +87,440 @@ func (s *conn) closeProcess() { } func (s *conn) LocalAddr() net.Addr { - return s.mux.conn.LocalAddr() + return s.receiveWindow.mux.conn.LocalAddr() } func (s *conn) RemoteAddr() net.Addr { - return s.mux.conn.RemoteAddr() + return s.receiveWindow.mux.conn.RemoteAddr() } func (s *conn) SetDeadline(t time.Time) error { - s.readTimeOut = t - s.writeTimeOut = t + _ = s.SetReadDeadline(t) + _ = s.SetWriteDeadline(t) return nil } func (s *conn) SetReadDeadline(t time.Time) error { - s.readTimeOut = t + s.receiveWindow.SetTimeOut(t) return nil } func (s *conn) SetWriteDeadline(t time.Time) error { - s.writeTimeOut = t + s.sendWindow.SetTimeOut(t) return nil } type window struct { - windowBuff []byte - off uint16 - readOp chan struct{} - readWait bool - WindowFull bool - usableReceiveWindow chan uint16 - WriteWg sync.WaitGroup - closeOp bool - closeOpCh chan struct{} - WriteEndOp chan struct{} - mutex sync.Mutex + off uint32 + maxSize uint32 + closeOp bool + closeOpCh chan struct{} + mux *Mux } -func (Self *window) NewReceive() { +func (Self *window) New() { + Self.closeOpCh = make(chan struct{}, 2) +} + +func (Self *window) CloseWindow() { + if !Self.closeOp { + Self.closeOp = true + Self.closeOpCh <- struct{}{} + Self.closeOpCh <- struct{}{} + } +} + +type ReceiveWindow struct { + bufQueue FIFOQueue + element *ListElement + readLength uint32 + readOp chan struct{} + readWait bool + windowFull bool + count int8 + bw *bandwidth + once sync.Once + window +} + +func (Self *ReceiveWindow) New(mux *Mux) { // initial a window for receive - Self.windowBuff = common.WindowBuff.Get() Self.readOp = make(chan struct{}) - Self.WriteEndOp = make(chan struct{}) - Self.closeOpCh = make(chan struct{}, 3) + Self.bufQueue.New() + Self.bw = new(bandwidth) + Self.element = new(ListElement) + Self.maxSize = 8192 + Self.mux = mux + Self.window.New() } -func (Self *window) NewSend() { - // initial a window for send - Self.usableReceiveWindow = make(chan uint16) - Self.closeOpCh = make(chan struct{}, 3) +func (Self *ReceiveWindow) RemainingSize() (n uint32) { + // receive window remaining + if Self.maxSize >= Self.bufQueue.Len() { + n = Self.maxSize - Self.bufQueue.Len() + } + // if maxSize is small than bufQueue length, return 0 + return } -func (Self *window) SetSendBuf(buf []byte) { +func (Self *ReceiveWindow) ReadSize() (n uint32) { + // acknowledge the size already read + Self.bufQueue.mutex.Lock() + n = Self.readLength + Self.readLength = 0 + Self.bufQueue.mutex.Unlock() + Self.count += 1 + return +} + +func (Self *ReceiveWindow) CalcSize() { + // calculating maximum receive window size + if Self.count == 0 { + logs.Warn("ping, bw", Self.mux.latency, Self.bw.Get()) + n := uint32(2 * Self.mux.latency * Self.bw.Get()) + if n < 8192 { + n = 8192 + } + if n < Self.bufQueue.Len() { + n = Self.bufQueue.Len() + } + // set the minimal size + logs.Warn("n", n) + Self.maxSize = n + Self.count = -5 + } +} + +func (Self *ReceiveWindow) Write(buf []byte, l uint16, part bool, id int32) (err error) { + if Self.closeOp { + return errors.New("conn.receiveWindow: write on closed window") + } + element := ListElement{} + err = element.New(buf, l, part) + //logs.Warn("push the buf", len(buf), l, (&element).l) + if err != nil { + return + } + Self.bufQueue.Push(&element) // must push data before allow read + //logs.Warn("read session calc size ", Self.maxSize) + // calculating the receive window size + Self.CalcSize() + logs.Warn("read session calc size finish", Self.maxSize) + if Self.RemainingSize() == 0 { + Self.windowFull = true + //logs.Warn("window full true", Self.windowFull) + } + Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.ReadSize()) + return nil +} + +func (Self *ReceiveWindow) Read(p []byte, id int32) (n int, err error) { + if Self.closeOp { + return 0, io.EOF // receive close signal, returns eof + } + pOff := 0 + l := 0 + //logs.Warn("receive window read off, element.l", Self.off, Self.element.l) +copyData: + Self.bw.StartRead() + if Self.off == uint32(Self.element.l) { + // on the first Read method invoked, Self.off and Self.element.l + // both zero value + Self.element, err = Self.bufQueue.Pop() + // if the queue is empty, Pop method will wait until one element push + // into the queue successful, or timeout. + // timer start on timeout parameter is set up , + // reset to 60s if timeout and data still available + Self.off = 0 + if err != nil { + return // queue receive stop or time out, break the loop and return + } + //logs.Warn("pop element", Self.element.l, Self.element.part) + } + l = copy(p[pOff:], Self.element.buf[Self.off:]) + Self.bw.SetCopySize(l) + pOff += l + Self.off += uint32(l) + Self.bufQueue.mutex.Lock() + Self.readLength += uint32(l) + //logs.Warn("window read length buf len", Self.readLength, Self.bufQueue.Len()) + Self.bufQueue.mutex.Unlock() + n += l + l = 0 + Self.bw.EndRead() + Self.sendStatus(id) + if pOff < len(p) && Self.element.part { + // element is a part of the segments, trying to fill up buf p + goto copyData + } + return // buf p is full or all of segments in buf, return +} + +func (Self *ReceiveWindow) sendStatus(id int32) { + if Self.windowFull || Self.bufQueue.Len() == 0 { + // window is full before read or empty now + Self.windowFull = false + Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.ReadSize()) + // acknowledge other side, have empty some receive window space + //} + } +} + +func (Self *ReceiveWindow) SetTimeOut(t time.Time) { + // waiting for FIFO queue Pop method + Self.bufQueue.SetTimeOut(t) +} + +func (Self *ReceiveWindow) Stop() { + // queue has no more data to push, so unblock pop method + Self.once.Do(Self.bufQueue.Stop) +} + +func (Self *ReceiveWindow) CloseWindow() { + Self.window.CloseWindow() + Self.Stop() +} + +type SendWindow struct { + buf []byte + sentLength uint32 + setSizeCh chan struct{} + setSizeWait bool + unSlide uint32 + timeout time.Time + window + mutex sync.Mutex +} + +func (Self *SendWindow) New(mux *Mux) { + Self.setSizeCh = make(chan struct{}) + Self.maxSize = 4096 + Self.mux = mux + Self.window.New() +} + +func (Self *SendWindow) SetSendBuf(buf []byte) { // send window buff from conn write method, set it to send window Self.mutex.Lock() - Self.windowBuff = buf + Self.buf = buf Self.off = 0 Self.mutex.Unlock() } -func (Self *window) fullSlide() { - // slide by allocate - newBuf := common.WindowBuff.Get() - Self.liteSlide() - n := copy(newBuf[:Self.len()], Self.windowBuff) - common.WindowBuff.Put(Self.windowBuff) - Self.windowBuff = newBuf[:n] +func (Self *SendWindow) RemainingSize() (n uint32) { + if Self.maxSize >= Self.sentLength { + n = Self.maxSize - Self.sentLength + } return } -func (Self *window) liteSlide() { - // slide by re slice - Self.windowBuff = Self.windowBuff[Self.off:] - Self.off = 0 - return -} - -func (Self *window) Size() (n int) { - // receive Window remaining - n = common.PoolSizeWindow - Self.len() - return -} - -func (Self *window) len() (n int) { - n = len(Self.windowBuff[Self.off:]) - return -} - -func (Self *window) cap() (n int) { - n = cap(Self.windowBuff[Self.off:]) - return -} - -func (Self *window) grow(n int) { - Self.windowBuff = Self.windowBuff[:Self.len()+n] -} - -func (Self *window) Write(p []byte) (n int, err error) { - if Self.closeOp { - return 0, errors.New("conn.receiveWindow: write on closed window") - } - if len(p) > Self.Size() { - return 0, errors.New("conn.receiveWindow: write too large") - } - Self.mutex.Lock() - // slide the offset - if len(p) > Self.cap()-Self.len() { - // not enough space, need to allocate - Self.fullSlide() - } else { - // have enough space, re slice - Self.liteSlide() - } - length := Self.len() // length before grow - Self.grow(len(p)) // grow for copy - n = copy(Self.windowBuff[length:], p) // must copy data before allow Read - if Self.readWait { - // if there condition is length == 0 and - // Read method just take away all the windowBuff, - // this method will block until windowBuff is empty again - - // allow continue read - defer Self.allowRead() - } - Self.mutex.Unlock() - return n, nil -} - -func (Self *window) allowRead() (closed bool) { - if Self.closeOp { - close(Self.readOp) - return true - } - Self.mutex.Lock() - Self.readWait = false - Self.mutex.Unlock() - select { - case <-Self.closeOpCh: - close(Self.readOp) - return true - case Self.readOp <- struct{}{}: - return false - } -} - -func (Self *window) Read(p []byte) (n int, err error) { - if Self.closeOp { - return 0, io.EOF // Write method receive close signal, returns eof - } - Self.mutex.Lock() - length := Self.len() // protect the length data, it invokes - // before Write lock and after Write unlock - if length == 0 { - // window is empty, waiting for Write method send a success readOp signal - // or get timeout or close - Self.readWait = true - Self.mutex.Unlock() - ticker := time.NewTicker(2 * time.Minute) - defer ticker.Stop() - select { - case _, ok := <-Self.readOp: - if !ok { - return 0, errors.New("conn.receiveWindow: window closed") - } - case <-Self.WriteEndOp: - return 0, io.EOF // receive eof signal, returns eof - case <-ticker.C: - return 0, errors.New("conn.receiveWindow: read time out") - case <-Self.closeOpCh: - close(Self.readOp) - return 0, io.EOF // receive close signal, returns eof - } - } else { - Self.mutex.Unlock() - } - minCopy := 512 - for { - Self.mutex.Lock() - if len(p) == n || Self.len() == 0 { - Self.mutex.Unlock() - break - } - if n+minCopy > len(p) { - minCopy = len(p) - n - } - i := copy(p[n:n+minCopy], Self.windowBuff[Self.off:]) - Self.off += uint16(i) - n += i - Self.mutex.Unlock() - } - p = p[:n] - return -} - -func (Self *window) WriteTo() (p []byte, err error) { - if Self.closeOp { - return nil, errors.New("conn.writeWindow: window closed") - } - if Self.len() == 0 { - return nil, io.EOF - // send window buff is drain, return eof and get another one - } - var windowSize uint16 - var ok bool -waiting: - ticker := time.NewTicker(2 * time.Minute) - defer ticker.Stop() - // waiting for receive usable window size, or timeout - select { - case windowSize, ok = <-Self.usableReceiveWindow: - if !ok { - return nil, errors.New("conn.writeWindow: window closed") - } - case <-ticker.C: - return nil, errors.New("conn.writeWindow: write to time out") - case <-Self.closeOpCh: - return nil, errors.New("conn.writeWindow: window closed") - } - if windowSize == 0 { - goto waiting // waiting for another usable window size - } - Self.mutex.Lock() - if windowSize > uint16(Self.len()) { - // usable window size is bigger than window buff size, send the full buff - windowSize = uint16(Self.len()) - } - p = Self.windowBuff[Self.off : windowSize+Self.off] - Self.off += windowSize - Self.mutex.Unlock() - return -} - -func (Self *window) SetAllowSize(value uint16) (closed bool) { +func (Self *SendWindow) SetSize(windowSize, readLength uint32) (closed bool) { defer func() { if recover() != nil { closed = true } }() if Self.closeOp { - close(Self.usableReceiveWindow) + close(Self.setSizeCh) return true } - select { - case Self.usableReceiveWindow <- value: - return false - case <-Self.closeOpCh: - close(Self.usableReceiveWindow) - return true + if readLength == 0 && Self.maxSize == windowSize { + logs.Warn("waiting for another window size") + return false // waiting for receive another usable window size } + logs.Warn("set send window size to ", windowSize, readLength) + Self.mutex.Lock() + Self.slide(windowSize, readLength) + if Self.setSizeWait { + // send window into the wait status, need notice the channel + //logs.Warn("send window remaining size is 0 , wait") + if Self.RemainingSize() == 0 { + //logs.Warn("waiting for another window size after slide") + // keep the wait status + Self.mutex.Unlock() + return false + } + Self.setSizeWait = false + Self.mutex.Unlock() + //logs.Warn("send window remaining size is 0 starting wait") + select { + case Self.setSizeCh <- struct{}{}: + //logs.Warn("send window remaining size is 0 finish") + return false + case <-Self.closeOpCh: + close(Self.setSizeCh) + return true + } + } + // send window not into the wait status, so just do slide + Self.mutex.Unlock() + return false } -func (Self *window) CloseWindow() { - Self.closeOp = true - Self.closeOpCh <- struct{}{} - Self.closeOpCh <- struct{}{} - Self.closeOpCh <- struct{}{} - close(Self.closeOpCh) +func (Self *SendWindow) slide(windowSize, readLength uint32) { + Self.sentLength -= readLength + Self.maxSize = windowSize +} + +func (Self *SendWindow) WriteTo() (p []byte, part bool, err error) { + // returns buf segments, return only one segments, need a loop outside + // until err = io.EOF + if Self.closeOp { + return nil, false, errors.New("conn.writeWindow: window closed") + } + if Self.off == uint32(len(Self.buf)) { + return nil, false, io.EOF + // send window buff is drain, return eof and get another one + } + Self.mutex.Lock() + if Self.RemainingSize() == 0 { + Self.setSizeWait = true + Self.mutex.Unlock() + // into the wait status + err = Self.waitReceiveWindow() + if err != nil { + return nil, false, err + } + } else { + Self.mutex.Unlock() + } + Self.mutex.Lock() + var sendSize uint32 + if len(Self.buf[Self.off:]) > common.MAXIMUM_SEGMENT_SIZE { + sendSize = common.MAXIMUM_SEGMENT_SIZE + part = true + } else { + sendSize = uint32(len(Self.buf[Self.off:])) + part = false + } + if Self.RemainingSize() < sendSize { + // usable window size is small than + // window MAXIMUM_SEGMENT_SIZE or send buf left + sendSize = Self.RemainingSize() + part = true + } + //logs.Warn("send size", sendSize) + p = Self.buf[Self.off : sendSize+Self.off] + Self.off += sendSize + Self.sentLength += sendSize + Self.mutex.Unlock() return } -type waitingCh struct { - nCh chan int - errCh chan error +func (Self *SendWindow) waitReceiveWindow() (err error) { + t := Self.timeout.Sub(time.Now()) + if t < 0 { + t = time.Minute + } + timer := time.NewTimer(t) + defer timer.Stop() + // waiting for receive usable window size, or timeout + select { + case _, ok := <-Self.setSizeCh: + if !ok { + return errors.New("conn.writeWindow: window closed") + } + return nil + case <-timer.C: + return errors.New("conn.writeWindow: write to time out") + case <-Self.closeOpCh: + return errors.New("conn.writeWindow: window closed") + } } -func (Self *waitingCh) new() { - Self.nCh = make(chan int) - Self.errCh = make(chan error) +func (Self *SendWindow) WriteFull(buf []byte, id int32) (n int, err error) { + Self.SetSendBuf(buf) // set the buf to send window + var bufSeg []byte + var part bool + for { + bufSeg, part, err = Self.WriteTo() + //logs.Warn("buf seg", len(bufSeg), part, err) + // get the buf segments from send window + if bufSeg == nil && part == false && err == io.EOF { + // send window is drain, break the loop + err = nil + break + } + if err != nil { + break + } + n += len(bufSeg) + if part { + Self.mux.sendInfo(common.MUX_NEW_MSG_PART, id, bufSeg) + } else { + Self.mux.sendInfo(common.MUX_NEW_MSG, id, bufSeg) + //logs.Warn("buf seg sent", len(bufSeg), part, err) + } + // send to other side, not send nil data to other side + } + //logs.Warn("buf seg write success") + return } -func (Self *waitingCh) close() { - close(Self.nCh) - close(Self.errCh) +func (Self *SendWindow) SetTimeOut(t time.Time) { + // waiting for receive a receive window size + Self.timeout = t +} + +type bandwidth struct { + lastReadStart time.Time + readStart time.Time + readEnd time.Time + bufLength int + lastBufLength int + count int8 + readBW float64 + writeBW float64 +} + +func (Self *bandwidth) StartRead() { + Self.lastReadStart, Self.readStart = Self.readStart, time.Now() +} + +func (Self *bandwidth) EndRead() { + if !Self.lastReadStart.IsZero() { + if Self.count == 0 { + Self.calcWriteBandwidth() + } + } + Self.readEnd = time.Now() + if Self.count == 0 { + Self.calcReadBandwidth() + Self.count = -3 + } + Self.count += 1 +} + +func (Self *bandwidth) SetCopySize(n int) { + // must be invoke between StartRead and EndRead + Self.lastBufLength, Self.bufLength = Self.bufLength, n +} + +func (Self *bandwidth) calcReadBandwidth() { + // Bandwidth between nps and npc + readTime := Self.readEnd.Sub(Self.readStart) + Self.readBW = float64(Self.bufLength) / readTime.Seconds() + //logs.Warn("calc read bw", Self.bufLength, readTime.Seconds()) +} + +func (Self *bandwidth) calcWriteBandwidth() { + // Bandwidth between nps and user, npc and application + //logs.Warn("calc write bw") + writeTime := Self.readEnd.Sub(Self.lastReadStart) + Self.writeBW = float64(Self.lastBufLength) / writeTime.Seconds() +} + +func (Self *bandwidth) Get() (bw float64) { + // The zero value, 0 for numeric types + if Self.writeBW == 0 && Self.readBW == 0 { + logs.Warn("bw both 0") + return 100 + } + if Self.writeBW == 0 && Self.readBW != 0 { + return Self.readBW + } + if Self.readBW == 0 && Self.writeBW != 0 { + return Self.writeBW + } + return math.Min(Self.readBW, Self.writeBW) } diff --git a/lib/mux/mux.go b/lib/mux/mux.go index a662ad0..e6a9e67 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -3,6 +3,7 @@ package mux import ( "bytes" "errors" + "io" "math" "net" "sync" @@ -22,8 +23,10 @@ type Mux struct { closeChan chan struct{} IsClose bool pingOk int + latency float64 + pingCh chan []byte connType string - writeQueue Queue + writeQueue PriorityQueue bufCh chan *bytes.Buffer sync.Mutex } @@ -38,13 +41,15 @@ func NewMux(c net.Conn, connType string) *Mux { IsClose: false, connType: connType, bufCh: make(chan *bytes.Buffer), + pingCh: make(chan []byte), } m.writeQueue.New() //read session by flag - go m.readSession() + m.readSession() //ping - go m.ping() - go m.writeSession() + m.ping() + m.pingReturn() + m.writeSession() return m } @@ -83,10 +88,10 @@ func (s *Mux) Addr() net.Addr { return s.conn.LocalAddr() } -func (s *Mux) sendInfo(flag uint8, id int32, data interface{}) { +func (s *Mux) sendInfo(flag uint8, id int32, data ...interface{}) { var err error pack := common.MuxPack.Get() - err = pack.NewPac(flag, id, data) + err = pack.NewPac(flag, id, data...) if err != nil { common.MuxPack.Put(pack) return @@ -98,11 +103,13 @@ func (s *Mux) sendInfo(flag uint8, id int32, data interface{}) { func (s *Mux) writeSession() { go s.packBuf() go s.writeBuf() - <-s.closeChan } func (s *Mux) packBuf() { for { + if s.IsClose { + break + } pack := s.writeQueue.Pop() buffer := common.BuffPool.Get() err := pack.Pack(buffer) @@ -117,12 +124,14 @@ func (s *Mux) packBuf() { case <-s.closeChan: break } - } } func (s *Mux) writeBuf() { for { + if s.IsClose { + break + } select { case buffer := <-s.bufCh: l := buffer.Len() @@ -141,8 +150,15 @@ func (s *Mux) writeBuf() { func (s *Mux) ping() { go func() { - ticker := time.NewTicker(time.Second * 1) + now, _ := time.Now().MarshalText() + s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) + // send the ping flag and get the latency first + ticker := time.NewTicker(time.Second * 15) for { + if s.IsClose { + ticker.Stop() + break + } select { case <-ticker.C: } @@ -150,7 +166,8 @@ func (s *Mux) ping() { if (math.MaxInt32 - s.id) < 10000 { s.id = 0 } - s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, nil) + now, _ := time.Now().MarshalText() + s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) if s.pingOk > 10 && s.connType == "kcp" { s.Close() break @@ -158,15 +175,32 @@ func (s *Mux) ping() { s.pingOk++ } }() - select { - case <-s.closeChan: - } +} + +func (s *Mux) pingReturn() { + go func() { + var now time.Time + var data []byte + for { + select { + case data = <-s.pingCh: + case <-s.closeChan: + break + } + _ = now.UnmarshalText(data) + s.latency = time.Since(now).Seconds() + s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, nil) + } + }() } func (s *Mux) readSession() { go func() { pack := common.MuxPack.Get() for { + if s.IsClose { + break + } pack = common.MuxPack.Get() if pack.UnPack(s.conn) != nil { break @@ -176,44 +210,25 @@ func (s *Mux) readSession() { case common.MUX_NEW_CONN: //new connection connection := NewConn(pack.Id, s) s.connMap.Set(pack.Id, connection) //it has been set before send ok - go func(connection *conn) { - connection.sendWindow.SetAllowSize(512) // set the initial receive window - }(connection) s.newConnCh <- connection s.sendInfo(common.MUX_NEW_CONN_OK, connection.connId, nil) continue case common.MUX_PING_FLAG: //ping - go s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, nil) + s.pingCh <- pack.Content continue case common.MUX_PING_RETURN: continue } if connection, ok := s.connMap.Get(pack.Id); ok && !connection.isClose { switch pack.Flag { - case common.MUX_NEW_MSG: //new msg from remote connection - //insert wait queue - if connection.isClose { - continue + case common.MUX_NEW_MSG, common.MUX_NEW_MSG_PART: //new msg from remote connection + err := s.newMsg(connection, pack) + if err != nil { + connection.Close() } - connection.receiveWindow.WriteWg.Add(1) - go func(connection *conn, content []byte) { // do not block read session - _, err := connection.receiveWindow.Write(content) - if err != nil { - logs.Warn("mux new msg err close", err) - connection.Close() - } - size := connection.receiveWindow.Size() - if size == 0 { - connection.receiveWindow.WindowFull = true - } - s.sendInfo(common.MUX_MSG_SEND_OK, connection.connId, size) - connection.receiveWindow.WriteWg.Done() - }(connection, pack.Content) continue case common.MUX_NEW_CONN_OK: //connection ok connection.connStatusOkCh <- struct{}{} - go connection.sendWindow.SetAllowSize(512) - // set the initial receive window both side continue case common.MUX_NEW_CONN_Fail: connection.connStatusFailCh <- struct{}{} @@ -222,15 +237,12 @@ func (s *Mux) readSession() { if connection.isClose { continue } - go connection.sendWindow.SetAllowSize(pack.Window) + connection.sendWindow.SetSize(pack.Window, pack.ReadLength) continue case common.MUX_CONN_CLOSE: //close the connection s.connMap.Delete(pack.Id) connection.closeFlag = true - go func(connection *conn) { - connection.receiveWindow.WriteWg.Wait() - connection.receiveWindow.WriteEndOp <- struct{}{} // close signal to receive window - }(connection) + connection.receiveWindow.Stop() // close signal to receive window continue } } else if pack.Flag == common.MUX_CONN_CLOSE { @@ -241,9 +253,24 @@ func (s *Mux) readSession() { common.MuxPack.Put(pack) s.Close() }() - select { - case <-s.closeChan: +} + +func (s *Mux) newMsg(connection *conn, pack *common.MuxPackager) (err error) { + if connection.isClose { + err = io.ErrClosedPipe + return } + //logs.Warn("read session receive new msg", pack.Length) + //go func(connection *conn, pack *common.MuxPackager) { // do not block read session + //insert into queue + if pack.Flag == common.MUX_NEW_MSG_PART { + err = connection.receiveWindow.Write(pack.Content, pack.Length, true, pack.Id) + } + if pack.Flag == common.MUX_NEW_MSG { + err = connection.receiveWindow.Write(pack.Content, pack.Length, false, pack.Id) + } + //logs.Warn("read session write success", pack.Length) + return } func (s *Mux) Close() error { @@ -255,9 +282,6 @@ func (s *Mux) Close() error { s.connMap.Close() s.closeChan <- struct{}{} s.closeChan <- struct{}{} - s.closeChan <- struct{}{} - s.closeChan <- struct{}{} - s.closeChan <- struct{}{} close(s.newConnCh) return s.conn.Close() } diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 081b2c9..5a57151 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -2,25 +2,54 @@ package mux import ( "container/list" + "errors" "github.com/cnlh/nps/lib/common" + "io" "sync" + "time" ) -type Queue struct { - list *list.List +type QueueOp struct { readOp chan struct{} cleanOp chan struct{} popWait bool mutex sync.Mutex } -func (Self *Queue) New() { - Self.list = list.New() +func (Self *QueueOp) New() { Self.readOp = make(chan struct{}) Self.cleanOp = make(chan struct{}, 2) } -func (Self *Queue) Push(packager *common.MuxPackager) { +func (Self *QueueOp) allowPop() (closed bool) { + Self.mutex.Lock() + Self.popWait = false + Self.mutex.Unlock() + select { + case Self.readOp <- struct{}{}: + return false + case <-Self.cleanOp: + return true + } +} + +func (Self *QueueOp) Clean() { + Self.cleanOp <- struct{}{} + Self.cleanOp <- struct{}{} + close(Self.cleanOp) +} + +type PriorityQueue struct { + list *list.List + QueueOp +} + +func (Self *PriorityQueue) New() { + Self.list = list.New() + Self.QueueOp.New() +} + +func (Self *PriorityQueue) Push(packager *common.MuxPackager) { Self.mutex.Lock() if Self.popWait { defer Self.allowPop() @@ -35,28 +64,16 @@ func (Self *Queue) Push(packager *common.MuxPackager) { return } -func (Self *Queue) allowPop() (closed bool) { - Self.mutex.Lock() - Self.popWait = false - Self.mutex.Unlock() - select { - case Self.readOp <- struct{}{}: - return false - case <-Self.cleanOp: - return true - } -} - -func (Self *Queue) insert(packager *common.MuxPackager) { +func (Self *PriorityQueue) insert(packager *common.MuxPackager) { element := Self.list.Back() for { - if element == nil { // Queue dose not have any of msg package with this close package id + if element == nil { // PriorityQueue dose not have any of msg package with this close package id Self.list.PushFront(packager) // insert close package to first break } if element.Value.(*common.MuxPackager).Flag == common.MUX_NEW_MSG && element.Value.(*common.MuxPackager).Id == packager.Id { - Self.list.InsertAfter(packager, element) // Queue has some msg package + Self.list.InsertAfter(packager, element) // PriorityQueue has some msg package // with this close package id, insert close package after last msg package break } @@ -64,7 +81,7 @@ func (Self *Queue) insert(packager *common.MuxPackager) { } } -func (Self *Queue) Pop() (packager *common.MuxPackager) { +func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { Self.mutex.Lock() element := Self.list.Front() if element != nil { @@ -73,7 +90,7 @@ func (Self *Queue) Pop() (packager *common.MuxPackager) { Self.mutex.Unlock() return } - Self.popWait = true // Queue is empty, notice Push method + Self.popWait = true // PriorityQueue is empty, notice Push method Self.mutex.Unlock() select { case <-Self.readOp: @@ -83,13 +100,90 @@ func (Self *Queue) Pop() (packager *common.MuxPackager) { } } -func (Self *Queue) Len() (n int) { +func (Self *PriorityQueue) Len() (n int) { n = Self.list.Len() return } -func (Self *Queue) Clean() { - Self.cleanOp <- struct{}{} - Self.cleanOp <- struct{}{} - close(Self.cleanOp) +type ListElement struct { + buf []byte + l uint16 + part bool +} + +func (Self *ListElement) New(buf []byte, l uint16, part bool) (err error) { + if uint16(len(buf)) != l { + return errors.New("ListElement: buf length not match") + } + Self.buf = buf + Self.l = l + Self.part = part + return nil +} + +type FIFOQueue struct { + list []*ListElement + length uint32 + stopOp chan struct{} + timeout time.Time + QueueOp +} + +func (Self *FIFOQueue) New() { + Self.QueueOp.New() + Self.stopOp = make(chan struct{}, 1) +} + +func (Self *FIFOQueue) Push(element *ListElement) { + Self.mutex.Lock() + if Self.popWait { + defer Self.allowPop() + } + Self.list = append(Self.list, element) + Self.length += uint32(element.l) + Self.mutex.Unlock() + return +} + +func (Self *FIFOQueue) Pop() (element *ListElement, err error) { + Self.mutex.Lock() + if len(Self.list) == 0 { + Self.popWait = true + Self.mutex.Unlock() + t := Self.timeout.Sub(time.Now()) + if t <= 0 { + t = time.Minute + } + timer := time.NewTimer(t) + defer timer.Stop() + select { + case <-Self.readOp: + Self.mutex.Lock() + case <-Self.cleanOp: + return + case <-Self.stopOp: + err = io.EOF + return + case <-timer.C: + err = errors.New("mux.queue: read time out") + return + } + } + element = Self.list[0] + Self.list = Self.list[1:] + Self.length -= uint32(element.l) + Self.mutex.Unlock() + return +} + +func (Self *FIFOQueue) Len() (n uint32) { + return Self.length +} + +func (Self *FIFOQueue) Stop() { + Self.stopOp <- struct{}{} +} + +func (Self *FIFOQueue) SetTimeOut(t time.Time) { + Self.timeout = t } From d9f9dc6acbbe199ff320df2444d375db6835ac69 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Tue, 8 Oct 2019 13:38:42 +0800 Subject: [PATCH 18/38] change ping calculate, fix window size calculate --- lib/common/netpackager.go | 20 ++++++++++---------- lib/mux/conn.go | 15 +++++++-------- lib/mux/mux.go | 14 +++++++++----- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/lib/common/netpackager.go b/lib/common/netpackager.go index ec2cb69..69ce96a 100644 --- a/lib/common/netpackager.go +++ b/lib/common/netpackager.go @@ -157,11 +157,11 @@ type MuxPackager struct { func (Self *MuxPackager) NewPac(flag uint8, id int32, content ...interface{}) (err error) { Self.Flag = flag Self.Id = id - if flag == MUX_NEW_MSG || flag == MUX_NEW_MSG_PART || flag == MUX_PING_FLAG { + switch flag { + case MUX_NEW_MSG, MUX_NEW_MSG_PART, MUX_PING_FLAG, MUX_PING_RETURN: err = Self.BasePackager.NewPac(content...) - } - if flag == MUX_MSG_SEND_OK { - // MUX_MSG_SEND_OK only allows one data + case MUX_MSG_SEND_OK: + // MUX_MSG_SEND_OK contains two data switch content[0].(type) { case int: Self.Window = uint32(content[0].(int)) @@ -187,10 +187,10 @@ func (Self *MuxPackager) Pack(writer io.Writer) (err error) { if err != nil { return } - if Self.Flag == MUX_NEW_MSG || Self.Flag == MUX_NEW_MSG_PART || Self.Flag == MUX_PING_FLAG { + switch Self.Flag { + case MUX_NEW_MSG, MUX_NEW_MSG_PART, MUX_PING_FLAG, MUX_PING_RETURN: err = Self.BasePackager.Pack(writer) - } - if Self.Flag == MUX_MSG_SEND_OK { + case MUX_MSG_SEND_OK: err = binary.Write(writer, binary.LittleEndian, Self.Window) if err != nil { return @@ -210,10 +210,10 @@ func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { if err != nil { return } - if Self.Flag == MUX_NEW_MSG || Self.Flag == MUX_NEW_MSG_PART || Self.Flag == MUX_PING_FLAG { + switch Self.Flag { + case MUX_NEW_MSG, MUX_NEW_MSG_PART, MUX_PING_FLAG, MUX_PING_RETURN: err = Self.BasePackager.UnPack(reader) - } - if Self.Flag == MUX_MSG_SEND_OK { + case MUX_MSG_SEND_OK: err = binary.Read(reader, binary.LittleEndian, &Self.Window) if err != nil { return diff --git a/lib/mux/conn.go b/lib/mux/conn.go index f4d5396..5dd69ea 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -2,7 +2,6 @@ package mux import ( "errors" - "github.com/astaxie/beego/logs" "io" "math" "net" @@ -169,14 +168,13 @@ func (Self *ReceiveWindow) ReadSize() (n uint32) { n = Self.readLength Self.readLength = 0 Self.bufQueue.mutex.Unlock() - Self.count += 1 return } func (Self *ReceiveWindow) CalcSize() { // calculating maximum receive window size if Self.count == 0 { - logs.Warn("ping, bw", Self.mux.latency, Self.bw.Get()) + //logs.Warn("ping, bw", Self.mux.latency, Self.bw.Get()) n := uint32(2 * Self.mux.latency * Self.bw.Get()) if n < 8192 { n = 8192 @@ -185,10 +183,11 @@ func (Self *ReceiveWindow) CalcSize() { n = Self.bufQueue.Len() } // set the minimal size - logs.Warn("n", n) + //logs.Warn("n", n) Self.maxSize = n Self.count = -5 } + Self.count += 1 } func (Self *ReceiveWindow) Write(buf []byte, l uint16, part bool, id int32) (err error) { @@ -205,7 +204,7 @@ func (Self *ReceiveWindow) Write(buf []byte, l uint16, part bool, id int32) (err //logs.Warn("read session calc size ", Self.maxSize) // calculating the receive window size Self.CalcSize() - logs.Warn("read session calc size finish", Self.maxSize) + //logs.Warn("read session calc size finish", Self.maxSize) if Self.RemainingSize() == 0 { Self.windowFull = true //logs.Warn("window full true", Self.windowFull) @@ -325,10 +324,10 @@ func (Self *SendWindow) SetSize(windowSize, readLength uint32) (closed bool) { return true } if readLength == 0 && Self.maxSize == windowSize { - logs.Warn("waiting for another window size") + //logs.Warn("waiting for another window size") return false // waiting for receive another usable window size } - logs.Warn("set send window size to ", windowSize, readLength) + //logs.Warn("set send window size to ", windowSize, readLength) Self.mutex.Lock() Self.slide(windowSize, readLength) if Self.setSizeWait { @@ -513,7 +512,7 @@ func (Self *bandwidth) calcWriteBandwidth() { func (Self *bandwidth) Get() (bw float64) { // The zero value, 0 for numeric types if Self.writeBW == 0 && Self.readBW == 0 { - logs.Warn("bw both 0") + //logs.Warn("bw both 0") return 100 } if Self.writeBW == 0 && Self.readBW != 0 { diff --git a/lib/mux/mux.go b/lib/mux/mux.go index e6a9e67..9023b82 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -150,7 +150,7 @@ func (s *Mux) writeBuf() { func (s *Mux) ping() { go func() { - now, _ := time.Now().MarshalText() + now, _ := time.Now().UTC().MarshalText() s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) // send the ping flag and get the latency first ticker := time.NewTicker(time.Second * 15) @@ -166,7 +166,7 @@ func (s *Mux) ping() { if (math.MaxInt32 - s.id) < 10000 { s.id = 0 } - now, _ := time.Now().MarshalText() + now, _ := time.Now().UTC().MarshalText() s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) if s.pingOk > 10 && s.connType == "kcp" { s.Close() @@ -188,8 +188,11 @@ func (s *Mux) pingReturn() { break } _ = now.UnmarshalText(data) - s.latency = time.Since(now).Seconds() - s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, nil) + s.latency = time.Now().UTC().Sub(now).Seconds() / 2 + //logs.Warn("latency", s.latency) + if s.latency <= 0 { + logs.Warn("latency err", s.latency) + } } }() } @@ -214,9 +217,10 @@ func (s *Mux) readSession() { s.sendInfo(common.MUX_NEW_CONN_OK, connection.connId, nil) continue case common.MUX_PING_FLAG: //ping - s.pingCh <- pack.Content + s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, pack.Content) continue case common.MUX_PING_RETURN: + s.pingCh <- pack.Content continue } if connection, ok := s.connMap.Get(pack.Id); ok && !connection.isClose { From 4c8d7b0738f492db4904c5f9c976359178dc93e0 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Tue, 8 Oct 2019 21:41:25 +0800 Subject: [PATCH 19/38] reduce memory allocate --- lib/common/const.go | 2 +- lib/common/pool.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/common/const.go b/lib/common/const.go index 95364a2..b4eac1a 100644 --- a/lib/common/const.go +++ b/lib/common/const.go @@ -48,5 +48,5 @@ const ( MUX_CONN_CLOSE MUX_PING_RETURN MUX_PING int32 = -1 - MAXIMUM_SEGMENT_SIZE = 4096 - 16 - 32 - 32 - 8 + MAXIMUM_SEGMENT_SIZE = PoolSizeWindow ) diff --git a/lib/common/pool.go b/lib/common/pool.go index 98da5d3..24efc60 100644 --- a/lib/common/pool.go +++ b/lib/common/pool.go @@ -9,7 +9,8 @@ const PoolSize = 64 * 1024 const PoolSizeSmall = 100 const PoolSizeUdp = 1472 const PoolSizeCopy = 32 << 10 -const PoolSizeWindow = 1<<16 - 1 +const PoolSizeBuffer = 4096 +const PoolSizeWindow = PoolSizeBuffer - 16 - 32 - 32 - 8 var BufPool = sync.Pool{ New: func() interface{} { @@ -92,18 +93,18 @@ type windowBufferPool struct { func (Self *windowBufferPool) New() { Self.pool = sync.Pool{ New: func() interface{} { - return make([]byte, 0, PoolSizeWindow) + return make([]byte, PoolSizeWindow, PoolSizeWindow) }, } } func (Self *windowBufferPool) Get() (buf []byte) { buf = Self.pool.Get().([]byte) - return buf[:0] + return buf[:PoolSizeWindow] } func (Self *windowBufferPool) Put(x []byte) { - if cap(x) == PoolSizeWindow { + if len(x) == PoolSizeWindow { Self.pool.Put(x[:PoolSizeWindow]) // make buf to full } else { x = nil @@ -117,7 +118,7 @@ type bufferPool struct { func (Self *bufferPool) New() { Self.pool = sync.Pool{ New: func() interface{} { - return new(bytes.Buffer) + return bytes.NewBuffer(make([]byte, 0, PoolSizeBuffer)) }, } } @@ -146,13 +147,12 @@ func (Self *muxPackagerPool) New() { func (Self *muxPackagerPool) Get() *MuxPackager { pack := Self.pool.Get().(*MuxPackager) - buf := CopyBuff.Get() - pack.Content = buf + pack.Content = WindowBuff.Get() return pack } func (Self *muxPackagerPool) Put(pack *MuxPackager) { - CopyBuff.Put(pack.Content) + WindowBuff.Put(pack.Content) Self.pool.Put(pack) } From 1f8e4410906a8e5a557535078818e22dff7bf35f Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sat, 12 Oct 2019 22:56:37 +0800 Subject: [PATCH 20/38] multiple changes --- README.md | 11 ++++++++- client/client.go | 5 ++++ client/control.go | 9 +++++-- cmd/nps/nps.go | 2 +- lib/common/const.go | 1 + lib/common/netpackager.go | 9 +++++-- lib/common/pool.go | 11 ++------- lib/common/util.go | 3 +++ lib/install/install.go | 38 +++++++++++++++++++++++++--- lib/mux/conn.go | 8 ++++++ lib/mux/mux.go | 17 +++++++++++-- lib/mux/mux_test.go | 52 +++++++++++++++++++++++++++++++++++++++ lib/mux/queue.go | 39 ++++++++++++++++++++--------- lib/version/version.go | 4 +-- 14 files changed, 176 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 1dd7f77..f56c32a 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,9 @@ nps是一款轻量级、高性能、功能强大的**内网穿透**代理服务 - 在刚才创建的客户端隧道管理中添加一条socks5代理，填写监听的端口（8003），保存。 - 在外网环境的本机配置socks5代理(例如使用proxifier进行全局代理)，ip为公网服务器ip（1.1.1.1），端口为填写的监听端口(8003)，即可畅享内网了 +**注意** +经过socks5代理，当收到socks5数据包时socket已经是accept状态。表现是扫描端口全open，建立连接后短时间关闭。若想同内网表现一致，建议远程连接一台设备。 + ### http正向代理 **适用范围：** 在外网环境下使用http正向代理访问内网站点 @@ -375,7 +378,13 @@ server { ``` (./nps|nps.exe) install ``` -安装成功后，对于linux，darwin，将会把配置文件和静态文件放置于/etc/nps/，并将可执行文件nps复制到/usr/bin/nps或者/usr/local/bin/nps，安装成功后可在任何位置执行 +安装成功后，对于linux，darwin，将会把配置文件和静态文件放置于/etc/nps/，并将可执行文件nps复制到/usr/bin/nps或者/usr/local/bin/nps，安装成功后可在任何位置执行，同时也会添加systemd配置。 + +``` +sudo systemctl enable|disable|start|stop|restart|status nps +``` +systemd，带有开机自启，自动重启配置，当进程结束后15秒会启动，日志输出至/var/log/nps/nps.log。 +建议采用此方式启动，能够捕获panic信息，便于排查问题。 ``` nps test|start|stop|restart|status diff --git a/client/client.go b/client/client.go index 52da907..5bd0b01 100755 --- a/client/client.go +++ b/client/client.go @@ -49,6 +49,11 @@ retry: time.Sleep(time.Second * 5) goto retry } + if c == nil { + logs.Error("Error data from server, and will be reconnected in five seconds") + time.Sleep(time.Second * 5) + goto retry + } logs.Info("Successful connection with server %s", s.svrAddr) //monitor the connection go s.ping() diff --git a/client/control.go b/client/control.go index 5673f14..3260113 100644 --- a/client/control.go +++ b/client/control.go @@ -223,8 +223,13 @@ func NewConn(tp string, vkey string, server string, connType string, proxyUrl st if _, err := c.Write([]byte(crypt.Md5(version.GetVersion()))); err != nil { return nil, err } - if b, err := c.GetShortContent(32); err != nil || crypt.Md5(version.GetVersion()) != string(b) { - logs.Error("The client does not match the server version. The current version of the client is", version.GetVersion()) + b, err := c.GetShortContent(32) + if err != nil { + logs.Error(err) + return nil, err + } + if crypt.Md5(version.GetVersion()) != string(b) { + logs.Error("The client does not match the server version. The current core version of the client is", version.GetVersion()) return nil, err } if _, err := c.Write([]byte(common.Getverifyval(vkey))); err != nil { diff --git a/cmd/nps/nps.go b/cmd/nps/nps.go index f66fe66..22835a2 100644 --- a/cmd/nps/nps.go +++ b/cmd/nps/nps.go @@ -61,7 +61,7 @@ func main() { logs.Error("Getting bridge_port error", err) os.Exit(0) } - logs.Info("the version of server is %s ,allow client version to be %s", version.VERSION, version.GetVersion()) + logs.Info("the version of server is %s ,allow client core version to be %s", version.VERSION, version.GetVersion()) connection.InitConnectionService() crypt.InitTls(filepath.Join(common.GetRunPath(), "conf", "server.pem"), filepath.Join(common.GetRunPath(), "conf", "server.key")) tool.InitAllowPort() diff --git a/lib/common/const.go b/lib/common/const.go index b4eac1a..f57ce4f 100644 --- a/lib/common/const.go +++ b/lib/common/const.go @@ -49,4 +49,5 @@ const ( MUX_PING_RETURN MUX_PING int32 = -1 MAXIMUM_SEGMENT_SIZE = PoolSizeWindow + MAXIMUM_WINDOW_SIZE = 1<<31 - 1 ) diff --git a/lib/common/netpackager.go b/lib/common/netpackager.go index 69ce96a..567a48f 100644 --- a/lib/common/netpackager.go +++ b/lib/common/netpackager.go @@ -158,8 +158,10 @@ func (Self *MuxPackager) NewPac(flag uint8, id int32, content ...interface{}) (e Self.Flag = flag Self.Id = id switch flag { - case MUX_NEW_MSG, MUX_NEW_MSG_PART, MUX_PING_FLAG, MUX_PING_RETURN: + case MUX_PING_FLAG, MUX_PING_RETURN, MUX_NEW_MSG, MUX_NEW_MSG_PART: + Self.Content = WindowBuff.Get() err = Self.BasePackager.NewPac(content...) + //logs.Warn(Self.Length, string(Self.Content)) case MUX_MSG_SEND_OK: // MUX_MSG_SEND_OK contains two data switch content[0].(type) { @@ -190,6 +192,7 @@ func (Self *MuxPackager) Pack(writer io.Writer) (err error) { switch Self.Flag { case MUX_NEW_MSG, MUX_NEW_MSG_PART, MUX_PING_FLAG, MUX_PING_RETURN: err = Self.BasePackager.Pack(writer) + WindowBuff.Put(Self.Content) case MUX_MSG_SEND_OK: err = binary.Write(writer, binary.LittleEndian, Self.Window) if err != nil { @@ -201,7 +204,6 @@ func (Self *MuxPackager) Pack(writer io.Writer) (err error) { } func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { - Self.BasePackager.clean() // also clean the content err = binary.Read(reader, binary.LittleEndian, &Self.Flag) if err != nil { return @@ -212,7 +214,10 @@ func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { } switch Self.Flag { case MUX_NEW_MSG, MUX_NEW_MSG_PART, MUX_PING_FLAG, MUX_PING_RETURN: + Self.Content = WindowBuff.Get() // need get a window buf from pool + Self.BasePackager.clean() // also clean the content err = Self.BasePackager.UnPack(reader) + //logs.Warn("unpack", Self.Length, string(Self.Content)) case MUX_MSG_SEND_OK: err = binary.Read(reader, binary.LittleEndian, &Self.Window) if err != nil { diff --git a/lib/common/pool.go b/lib/common/pool.go index 24efc60..240f7f9 100644 --- a/lib/common/pool.go +++ b/lib/common/pool.go @@ -104,11 +104,7 @@ func (Self *windowBufferPool) Get() (buf []byte) { } func (Self *windowBufferPool) Put(x []byte) { - if len(x) == PoolSizeWindow { - Self.pool.Put(x[:PoolSizeWindow]) // make buf to full - } else { - x = nil - } + Self.pool.Put(x[:PoolSizeWindow]) // make buf to full } type bufferPool struct { @@ -146,13 +142,10 @@ func (Self *muxPackagerPool) New() { } func (Self *muxPackagerPool) Get() *MuxPackager { - pack := Self.pool.Get().(*MuxPackager) - pack.Content = WindowBuff.Get() - return pack + return Self.pool.Get().(*MuxPackager) } func (Self *muxPackagerPool) Put(pack *MuxPackager) { - WindowBuff.Put(pack.Content) Self.pool.Put(pack) } diff --git a/lib/common/util.go b/lib/common/util.go index dc9afbe..e3dfb4f 100755 --- a/lib/common/util.go +++ b/lib/common/util.go @@ -268,6 +268,9 @@ func CopyBuffer(dst io.Writer, src io.Reader) (written int64, err error) { defer CopyBuff.Put(buf) for { nr, er := src.Read(buf) + //if len(pr)>0 && pr[0] && nr > 50 { + // logs.Warn(string(buf[:50])) + //} if nr > 0 { nw, ew := dst.Write(buf[0:nr]) if nw > 0 { diff --git a/lib/install/install.go b/lib/install/install.go index 56f3cc5..24af9b9 100644 --- a/lib/install/install.go +++ b/lib/install/install.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "io/ioutil" "log" "os" "path/filepath" @@ -13,6 +14,23 @@ import ( ) func InstallNps() { + unit := `[Unit] +Description=nps - convenient proxy server +Documentation=https://github.com/cnlh/nps/ +After=network-online.target remote-fs.target nss-lookup.target +Wants=network-online.target` + service := `[Service] +Type=simple +KillMode=process +Restart=always +RestartSec=15s +StandardOutput=append:/var/log/nps/nps.log +ExecStartPre=/bin/echo 'Starting nps' +ExecStopPost=/bin/echo 'Stopping nps' +ExecStart=` + install := `[Install] +WantedBy=multi-user.target` + path := common.GetInstallPath() if common.FileExists(path) { log.Fatalf("the path %s has exist, does not support install", path) @@ -35,21 +53,35 @@ func InstallNps() { log.Fatalln(err) } else { os.Chmod("/usr/local/bin/nps", 0755) + service += "/usr/local/bin/nps" log.Println("Executable files have been copied to", "/usr/local/bin/nps") } } else { os.Chmod("/usr/bin/nps", 0755) + service += "/usr/bin/nps" log.Println("Executable files have been copied to", "/usr/bin/nps") } - + systemd := unit + "\n\n" + service + "\n\n" + install + _ = os.Remove("/usr/lib/systemd/system/nps.service") + err := ioutil.WriteFile("/usr/lib/systemd/system/nps.service", []byte(systemd), 0644) + if err != nil { + log.Println("Write systemd service err ", err) + } + _ = os.Mkdir("/var/log/nps", 644) } log.Println("install ok!") log.Println("Static files and configuration files in the current directory will be useless") log.Println("The new configuration file is located in", path, "you can edit them") if !common.IsWindows() { - log.Println("You can start with nps test|start|stop|restart|status anywhere") + log.Println(`You can start with: +sudo systemctl enable|disable|start|stop|restart|status nps +or: +nps test|start|stop|restart|status +anywhere!`) } else { - log.Println("You can copy executable files to any directory and start working with nps.exe test|start|stop|restart|status") + log.Println(`You can copy executable files to any directory and start working with: +nps.exe test|start|stop|restart|status +now!`) } } func MkidrDirAll(path string, v ...string) { diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 5dd69ea..c4b47f3 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -183,6 +183,10 @@ func (Self *ReceiveWindow) CalcSize() { n = Self.bufQueue.Len() } // set the minimal size + if n > common.MAXIMUM_WINDOW_SIZE { + n = common.MAXIMUM_WINDOW_SIZE + } + // set the maximum size //logs.Warn("n", n) Self.maxSize = n Self.count = -5 @@ -248,6 +252,10 @@ copyData: l = 0 Self.bw.EndRead() Self.sendStatus(id) + if Self.off == uint32(Self.element.l) { + //logs.Warn("put the element end ", string(Self.element.buf[:15])) + common.WindowBuff.Put(Self.element.buf) + } if pOff < len(p) && Self.element.part { // element is a part of the segments, trying to fill up buf p goto copyData diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 9023b82..529b7dc 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -25,6 +25,7 @@ type Mux struct { pingOk int latency float64 pingCh chan []byte + pingTimer *time.Timer connType string writeQueue PriorityQueue bufCh chan *bytes.Buffer @@ -42,6 +43,7 @@ func NewMux(c net.Conn, connType string) *Mux { connType: connType, bufCh: make(chan *bytes.Buffer), pingCh: make(chan []byte), + pingTimer: time.NewTimer(15 * time.Second), } m.writeQueue.New() //read session by flag @@ -119,6 +121,7 @@ func (s *Mux) packBuf() { common.BuffPool.Put(buffer) break } + //logs.Warn(buffer.String()) select { case s.bufCh <- buffer: case <-s.closeChan: @@ -153,7 +156,7 @@ func (s *Mux) ping() { now, _ := time.Now().UTC().MarshalText() s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) // send the ping flag and get the latency first - ticker := time.NewTicker(time.Second * 15) + ticker := time.NewTicker(time.Second * 5) for { if s.IsClose { ticker.Stop() @@ -168,6 +171,10 @@ func (s *Mux) ping() { } now, _ := time.Now().UTC().MarshalText() s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) + if !s.pingTimer.Stop() { + <-s.pingTimer.C + } + s.pingTimer.Reset(15 * time.Second) if s.pingOk > 10 && s.connType == "kcp" { s.Close() break @@ -186,10 +193,15 @@ func (s *Mux) pingReturn() { case data = <-s.pingCh: case <-s.closeChan: break + case <-s.pingTimer.C: + logs.Error("mux: ping time out") + s.Close() + break } _ = now.UnmarshalText(data) s.latency = time.Now().UTC().Sub(now).Seconds() / 2 - //logs.Warn("latency", s.latency) + logs.Warn("latency", s.latency) + common.WindowBuff.Put(data) if s.latency <= 0 { logs.Warn("latency err", s.latency) } @@ -218,6 +230,7 @@ func (s *Mux) readSession() { continue case common.MUX_PING_FLAG: //ping s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, pack.Content) + common.WindowBuff.Put(pack.Content) continue case common.MUX_PING_RETURN: s.pingCh <- pack.Content diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index c7b10e0..abc4eb4 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -1,8 +1,11 @@ package mux import ( + "bufio" + "fmt" "net" "net/http" + "net/http/httputil" _ "net/http/pprof" "sync" "testing" @@ -37,6 +40,7 @@ func TestNewMux(t *testing.T) { c2, err := net.Dial("tcp", "127.0.0.1:80") if err != nil { logs.Warn(err) + c.Close() continue } go func(c2 net.Conn, c net.Conn) { @@ -107,6 +111,9 @@ func TestNewMux(t *testing.T) { } }() + time.Sleep(time.Second * 5) + //go test_request() + for { time.Sleep(time.Second * 5) } @@ -135,6 +142,51 @@ func client() { } } +func test_request() { + conn, _ := net.Dial("tcp", "127.0.0.1:7777") + for { + conn.Write([]byte(`GET /videojs5/video.js HTTP/1.1 +Host: 127.0.0.1:7777 +Connection: keep-alive + + +`)) + r, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil { + logs.Warn("close by read response err", err) + break + } + logs.Warn("read response success", r) + b, err := httputil.DumpResponse(r, true) + if err != nil { + logs.Warn("close by dump response err", err) + break + } + fmt.Println(string(b[:20]), err) + time.Sleep(time.Second) + } +} + +func test_raw() { + conn, _ := net.Dial("tcp", "127.0.0.1:7777") + for { + conn.Write([]byte(`GET /videojs5/test HTTP/1.1 +Host: 127.0.0.1:7777 +Connection: keep-alive + + +`)) + buf := make([]byte, 1000000) + n, err := conn.Read(buf) + if err != nil { + logs.Warn("close by read response err", err) + break + } + logs.Warn(n, string(buf[:50]), "\n--------------\n", string(buf[n-50:n])) + time.Sleep(time.Second) + } +} + func TestNewConn(t *testing.T) { buf := common.GetBufPoolCopy() logs.Warn(len(buf), cap(buf)) diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 5a57151..a835e2a 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -51,15 +51,23 @@ func (Self *PriorityQueue) New() { func (Self *PriorityQueue) Push(packager *common.MuxPackager) { Self.mutex.Lock() - if Self.popWait { - defer Self.allowPop() - } - if packager.Flag == common.MUX_CONN_CLOSE { - Self.insert(packager) // the close package may need priority, - // prevent wait too long to close - } else { + switch packager.Flag { + case common.MUX_PING_FLAG, common.MUX_PING_RETURN: + Self.list.PushFront(packager) + // the ping package need highest priority + // prevent ping calculation error + case common.MUX_CONN_CLOSE: + Self.insert(packager) + // the close package may need priority too, set second + // prevent wait too long to close conn + default: Self.list.PushBack(packager) } + if Self.popWait { + Self.mutex.Unlock() + Self.allowPop() + return + } Self.mutex.Unlock() return } @@ -68,7 +76,14 @@ func (Self *PriorityQueue) insert(packager *common.MuxPackager) { element := Self.list.Back() for { if element == nil { // PriorityQueue dose not have any of msg package with this close package id - Self.list.PushFront(packager) // insert close package to first + element = Self.list.Front() + if element != nil { + Self.list.InsertAfter(packager, element) + // insert close package to second + } else { + Self.list.PushFront(packager) + // list is empty, push to front + } break } if element.Value.(*common.MuxPackager).Flag == common.MUX_NEW_MSG && @@ -136,11 +151,13 @@ func (Self *FIFOQueue) New() { func (Self *FIFOQueue) Push(element *ListElement) { Self.mutex.Lock() - if Self.popWait { - defer Self.allowPop() - } Self.list = append(Self.list, element) Self.length += uint32(element.l) + if Self.popWait { + Self.mutex.Unlock() + Self.allowPop() + return + } Self.mutex.Unlock() return } diff --git a/lib/version/version.go b/lib/version/version.go index 902b30b..4cc0532 100644 --- a/lib/version/version.go +++ b/lib/version/version.go @@ -1,8 +1,8 @@ package version -const VERSION = "0.23.2" +const VERSION = "0.23.3" // Compulsory minimum version, Minimum downward compatibility to this version func GetVersion() string { - return "0.21.0" + return "0.22.0" } From f5d5f633660d7ffb1325dbcb3a18bd2797431ea2 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sun, 13 Oct 2019 22:45:40 +0800 Subject: [PATCH 21/38] change slide window bandwidth calculation --- lib/common/netpackager.go | 16 ++-- lib/mux/conn.go | 160 +++++++++++++++++++++----------------- lib/mux/map.go | 4 + lib/mux/mux.go | 67 ++++++++++++++-- 4 files changed, 163 insertions(+), 84 deletions(-) diff --git a/lib/common/netpackager.go b/lib/common/netpackager.go index 567a48f..91eeb98 100644 --- a/lib/common/netpackager.go +++ b/lib/common/netpackager.go @@ -65,8 +65,9 @@ func (Self *BasePackager) Pack(writer io.Writer) (err error) { //Unpack 会导致传入的数字类型转化成float64！！ //主要原因是json unmarshal并未传入正确的数据类型 -func (Self *BasePackager) UnPack(reader io.Reader) (err error) { +func (Self *BasePackager) UnPack(reader io.Reader) (n uint16, err error) { Self.clean() + n += 2 // uint16 err = binary.Read(reader, binary.LittleEndian, &Self.Length) if err != nil { return @@ -80,6 +81,7 @@ func (Self *BasePackager) UnPack(reader io.Reader) (err error) { // err = io.ErrUnexpectedEOF //} err = binary.Read(reader, binary.LittleEndian, Self.Content) + n += Self.Length return } @@ -137,12 +139,13 @@ func (Self *ConnPackager) Pack(writer io.Writer) (err error) { return } -func (Self *ConnPackager) UnPack(reader io.Reader) (err error) { +func (Self *ConnPackager) UnPack(reader io.Reader) (n uint16, err error) { err = binary.Read(reader, binary.LittleEndian, &Self.ConnType) if err != nil && err != io.EOF { return } - err = Self.BasePackager.UnPack(reader) + n, err = Self.BasePackager.UnPack(reader) + n += 2 return } @@ -203,7 +206,7 @@ func (Self *MuxPackager) Pack(writer io.Writer) (err error) { return } -func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { +func (Self *MuxPackager) UnPack(reader io.Reader) (n uint16, err error) { err = binary.Read(reader, binary.LittleEndian, &Self.Flag) if err != nil { return @@ -216,14 +219,17 @@ func (Self *MuxPackager) UnPack(reader io.Reader) (err error) { case MUX_NEW_MSG, MUX_NEW_MSG_PART, MUX_PING_FLAG, MUX_PING_RETURN: Self.Content = WindowBuff.Get() // need get a window buf from pool Self.BasePackager.clean() // also clean the content - err = Self.BasePackager.UnPack(reader) + n, err = Self.BasePackager.UnPack(reader) //logs.Warn("unpack", Self.Length, string(Self.Content)) case MUX_MSG_SEND_OK: err = binary.Read(reader, binary.LittleEndian, &Self.Window) if err != nil { return } + n += 4 // uint32 err = binary.Read(reader, binary.LittleEndian, &Self.ReadLength) + n += 4 // uint32 } + n += 5 //uint8 int32 return } diff --git a/lib/mux/conn.go b/lib/mux/conn.go index c4b47f3..2d519cb 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -3,7 +3,6 @@ package mux import ( "errors" "io" - "math" "net" "sync" "time" @@ -137,8 +136,8 @@ type ReceiveWindow struct { readWait bool windowFull bool count int8 - bw *bandwidth - once sync.Once + //bw *bandwidth + once sync.Once window } @@ -146,7 +145,7 @@ func (Self *ReceiveWindow) New(mux *Mux) { // initial a window for receive Self.readOp = make(chan struct{}) Self.bufQueue.New() - Self.bw = new(bandwidth) + //Self.bw = new(bandwidth) Self.element = new(ListElement) Self.maxSize = 8192 Self.mux = mux @@ -175,7 +174,7 @@ func (Self *ReceiveWindow) CalcSize() { // calculating maximum receive window size if Self.count == 0 { //logs.Warn("ping, bw", Self.mux.latency, Self.bw.Get()) - n := uint32(2 * Self.mux.latency * Self.bw.Get()) + n := uint32(2 * Self.mux.latency * Self.mux.bw.Get() * 1.5 / float64(Self.mux.connMap.Size())) if n < 8192 { n = 8192 } @@ -183,13 +182,16 @@ func (Self *ReceiveWindow) CalcSize() { n = Self.bufQueue.Len() } // set the minimal size + if n > 2*Self.maxSize { + n = 2 * Self.maxSize + } if n > common.MAXIMUM_WINDOW_SIZE { n = common.MAXIMUM_WINDOW_SIZE } // set the maximum size //logs.Warn("n", n) Self.maxSize = n - Self.count = -5 + Self.count = -10 } Self.count += 1 } @@ -225,7 +227,7 @@ func (Self *ReceiveWindow) Read(p []byte, id int32) (n int, err error) { l := 0 //logs.Warn("receive window read off, element.l", Self.off, Self.element.l) copyData: - Self.bw.StartRead() + //Self.bw.StartRead() if Self.off == uint32(Self.element.l) { // on the first Read method invoked, Self.off and Self.element.l // both zero value @@ -241,7 +243,7 @@ copyData: //logs.Warn("pop element", Self.element.l, Self.element.part) } l = copy(p[pOff:], Self.element.buf[Self.off:]) - Self.bw.SetCopySize(l) + //Self.bw.SetCopySize(l) pOff += l Self.off += uint32(l) Self.bufQueue.mutex.Lock() @@ -250,7 +252,7 @@ copyData: Self.bufQueue.mutex.Unlock() n += l l = 0 - Self.bw.EndRead() + //Self.bw.EndRead() Self.sendStatus(id) if Self.off == uint32(Self.element.l) { //logs.Warn("put the element end ", string(Self.element.buf[:15])) @@ -469,65 +471,81 @@ func (Self *SendWindow) SetTimeOut(t time.Time) { Self.timeout = t } -type bandwidth struct { - lastReadStart time.Time - readStart time.Time - readEnd time.Time - bufLength int - lastBufLength int - count int8 - readBW float64 - writeBW float64 -} - -func (Self *bandwidth) StartRead() { - Self.lastReadStart, Self.readStart = Self.readStart, time.Now() -} - -func (Self *bandwidth) EndRead() { - if !Self.lastReadStart.IsZero() { - if Self.count == 0 { - Self.calcWriteBandwidth() - } - } - Self.readEnd = time.Now() - if Self.count == 0 { - Self.calcReadBandwidth() - Self.count = -3 - } - Self.count += 1 -} - -func (Self *bandwidth) SetCopySize(n int) { - // must be invoke between StartRead and EndRead - Self.lastBufLength, Self.bufLength = Self.bufLength, n -} - -func (Self *bandwidth) calcReadBandwidth() { - // Bandwidth between nps and npc - readTime := Self.readEnd.Sub(Self.readStart) - Self.readBW = float64(Self.bufLength) / readTime.Seconds() - //logs.Warn("calc read bw", Self.bufLength, readTime.Seconds()) -} - -func (Self *bandwidth) calcWriteBandwidth() { - // Bandwidth between nps and user, npc and application - //logs.Warn("calc write bw") - writeTime := Self.readEnd.Sub(Self.lastReadStart) - Self.writeBW = float64(Self.lastBufLength) / writeTime.Seconds() -} - -func (Self *bandwidth) Get() (bw float64) { - // The zero value, 0 for numeric types - if Self.writeBW == 0 && Self.readBW == 0 { - //logs.Warn("bw both 0") - return 100 - } - if Self.writeBW == 0 && Self.readBW != 0 { - return Self.readBW - } - if Self.readBW == 0 && Self.writeBW != 0 { - return Self.writeBW - } - return math.Min(Self.readBW, Self.writeBW) -} +//type bandwidth struct { +// readStart time.Time +// lastReadStart time.Time +// readEnd time.Time +// lastReadEnd time.Time +// bufLength int +// lastBufLength int +// count int8 +// readBW float64 +// writeBW float64 +// readBandwidth float64 +//} +// +//func (Self *bandwidth) StartRead() { +// Self.lastReadStart, Self.readStart = Self.readStart, time.Now() +// if !Self.lastReadStart.IsZero() { +// if Self.count == -5 { +// Self.calcBandWidth() +// } +// } +//} +// +//func (Self *bandwidth) EndRead() { +// Self.lastReadEnd, Self.readEnd = Self.readEnd, time.Now() +// if Self.count == -5 { +// Self.calcWriteBandwidth() +// } +// if Self.count == 0 { +// Self.calcReadBandwidth() +// Self.count = -6 +// } +// Self.count += 1 +//} +// +//func (Self *bandwidth) SetCopySize(n int) { +// // must be invoke between StartRead and EndRead +// Self.lastBufLength, Self.bufLength = Self.bufLength, n +//} +//// calculating +//// start end start end +//// read read +//// write +// +//func (Self *bandwidth) calcBandWidth() { +// t := Self.readStart.Sub(Self.lastReadStart) +// if Self.lastBufLength >= 32768 { +// Self.readBandwidth = float64(Self.lastBufLength) / t.Seconds() +// } +//} +// +//func (Self *bandwidth) calcReadBandwidth() { +// // Bandwidth between nps and npc +// readTime := Self.readEnd.Sub(Self.readStart) +// Self.readBW = float64(Self.bufLength) / readTime.Seconds() +// //logs.Warn("calc read bw", Self.readBW, Self.bufLength, readTime.Seconds()) +//} +// +//func (Self *bandwidth) calcWriteBandwidth() { +// // Bandwidth between nps and user, npc and application +// writeTime := Self.readStart.Sub(Self.lastReadEnd) +// Self.writeBW = float64(Self.lastBufLength) / writeTime.Seconds() +// //logs.Warn("calc write bw", Self.writeBW, Self.bufLength, writeTime.Seconds()) +//} +// +//func (Self *bandwidth) Get() (bw float64) { +// // The zero value, 0 for numeric types +// if Self.writeBW == 0 && Self.readBW == 0 { +// //logs.Warn("bw both 0") +// return 100 +// } +// if Self.writeBW == 0 && Self.readBW != 0 { +// return Self.readBW +// } +// if Self.readBW == 0 && Self.writeBW != 0 { +// return Self.writeBW +// } +// return Self.readBandwidth +//} diff --git a/lib/mux/map.go b/lib/mux/map.go index 0801201..8f07dee 100644 --- a/lib/mux/map.go +++ b/lib/mux/map.go @@ -20,6 +20,10 @@ func NewConnMap() *connMap { return connMap } +func (s *connMap) Size() (n int) { + return len(s.connMap) +} + func (s *connMap) Get(id int32) (*conn, bool) { s.Lock() defer s.Unlock() diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 529b7dc..42aa8c8 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -24,6 +24,7 @@ type Mux struct { IsClose bool pingOk int latency float64 + bw *bandwidth pingCh chan []byte pingTimer *time.Timer connType string @@ -37,8 +38,9 @@ func NewMux(c net.Conn, connType string) *Mux { conn: c, connMap: NewConnMap(), id: 0, - closeChan: make(chan struct{}), + closeChan: make(chan struct{}, 3), newConnCh: make(chan *conn), + bw: new(bandwidth), IsClose: false, connType: connType, bufCh: make(chan *bytes.Buffer), @@ -91,6 +93,9 @@ func (s *Mux) Addr() net.Addr { } func (s *Mux) sendInfo(flag uint8, id int32, data ...interface{}) { + if s.IsClose { + return + } var err error pack := common.MuxPack.Get() err = pack.NewPac(flag, id, data...) @@ -160,6 +165,9 @@ func (s *Mux) ping() { for { if s.IsClose { ticker.Stop() + if !s.pingTimer.Stop() { + <-s.pingTimer.C + } break } select { @@ -189,6 +197,9 @@ func (s *Mux) pingReturn() { var now time.Time var data []byte for { + if s.IsClose { + break + } select { case data = <-s.pingCh: case <-s.closeChan: @@ -199,12 +210,12 @@ func (s *Mux) pingReturn() { break } _ = now.UnmarshalText(data) - s.latency = time.Now().UTC().Sub(now).Seconds() / 2 - logs.Warn("latency", s.latency) - common.WindowBuff.Put(data) - if s.latency <= 0 { - logs.Warn("latency err", s.latency) + latency := time.Now().UTC().Sub(now).Seconds() / 2 + if latency < 0.5 && latency > 0 { + s.latency = latency } + //logs.Warn("latency", s.latency) + common.WindowBuff.Put(data) } }() } @@ -212,14 +223,18 @@ func (s *Mux) pingReturn() { func (s *Mux) readSession() { go func() { pack := common.MuxPack.Get() + var l uint16 + var err error for { if s.IsClose { break } pack = common.MuxPack.Get() - if pack.UnPack(s.conn) != nil { + s.bw.StartRead() + if l, err = pack.UnPack(s.conn); err != nil { break } + s.bw.SetCopySize(l) s.pingOk = 0 switch pack.Flag { case common.MUX_NEW_CONN: //new connection @@ -239,7 +254,7 @@ func (s *Mux) readSession() { if connection, ok := s.connMap.Get(pack.Id); ok && !connection.isClose { switch pack.Flag { case common.MUX_NEW_MSG, common.MUX_NEW_MSG_PART: //new msg from remote connection - err := s.newMsg(connection, pack) + err = s.newMsg(connection, pack) if err != nil { connection.Close() } @@ -299,6 +314,7 @@ func (s *Mux) Close() error { s.connMap.Close() s.closeChan <- struct{}{} s.closeChan <- struct{}{} + s.closeChan <- struct{}{} close(s.newConnCh) return s.conn.Close() } @@ -311,3 +327,38 @@ func (s *Mux) getId() (id int32) { } return } + +type bandwidth struct { + readStart time.Time + lastReadStart time.Time + bufLength uint16 + readBandwidth float64 +} + +func (Self *bandwidth) StartRead() { + if Self.readStart.IsZero() { + Self.readStart = time.Now() + } + if Self.bufLength >= 16384 { + Self.lastReadStart, Self.readStart = Self.readStart, time.Now() + Self.calcBandWidth() + } +} + +func (Self *bandwidth) SetCopySize(n uint16) { + Self.bufLength += n +} + +func (Self *bandwidth) calcBandWidth() { + t := Self.readStart.Sub(Self.lastReadStart) + Self.readBandwidth = float64(Self.bufLength) / t.Seconds() + Self.bufLength = 0 +} + +func (Self *bandwidth) Get() (bw float64) { + // The zero value, 0 for numeric types + if Self.readBandwidth <= 0 { + Self.readBandwidth = 100 + } + return Self.readBandwidth +} From 18ca5d04ccf2733c818160addb2f6526d483db5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=B2=B3?= Date: Mon, 14 Oct 2019 23:44:07 +0800 Subject: [PATCH 22/38] connection trace web server --- lib/mux/web.go | 97 +++++++++++++++++++++++++++++++++++++++++++++ lib/mux/web_test.go | 7 ++++ 2 files changed, 104 insertions(+) create mode 100644 lib/mux/web.go create mode 100644 lib/mux/web_test.go diff --git a/lib/mux/web.go b/lib/mux/web.go new file mode 100644 index 0000000..0ebe8e7 --- /dev/null +++ b/lib/mux/web.go @@ -0,0 +1,97 @@ +package mux + +import ( + "fmt" + "net/http" + "sort" + "strconv" + "time" +) + +type connLog struct { + startTime time.Time + isClose bool + logs []string +} + +var m map[int]*connLog + +var copyMap map[int]*connLog + +func deepCopyMap() { + copyMap = make(map[int]*connLog) + for k, v := range m { + copyMap[k] = &connLog{ + startTime: v.startTime, + isClose: v.isClose, + logs: v.logs, + } + } +} + +//func init() { +// m = make(map[int]*connLog) +// m[0] = &connLog{ +// startTime: time.Now(), +// isClose: false, +// logs: []string{"111", "222", "333"}, +// } +// m[1] = &connLog{ +// startTime: time.Now(), +// isClose: false, +// logs: []string{"111", "222", "333", "444"}, +// } +// m[2] = &connLog{ +// startTime: time.Now(), +// isClose: true, +// logs: []string{"111", "222", "333", "555"}, +// } +//} + +type IntSlice []int + +func (s IntSlice) Len() int { return len(s) } + +func (s IntSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func (s IntSlice) Less(i, j int) bool { return s[i] < s[j] } + +func NewLogServer() { + http.HandleFunc("/", index) + http.HandleFunc("/detail", detail) + http.HandleFunc("/stash", stash) + fmt.Println(http.ListenAndServe(":8899", nil)) +} + +func stash(w http.ResponseWriter, r *http.Request) { + deepCopyMap() + w.Write([]byte("ok")) +} + +func index(w http.ResponseWriter, r *http.Request) { + var keys []int + for k := range copyMap { + keys = append(keys, k) + } + sort.Sort(IntSlice(keys)) + var s string + for v := range keys { + connL := copyMap[v] + s += "" + strconv.Itoa(v) + "----------" + s += strconv.Itoa(int(time.Now().Unix()-connL.startTime.Unix())) + "s----------" + s += strconv.FormatBool(connL.isClose) + s += "
" + } + w.Write([]byte(s)) +} + +func detail(w http.ResponseWriter, r *http.Request) { + id := r.FormValue("id") + i, _ := strconv.Atoi(id) + v, _ := copyMap[i] + var s string + for _, vv := range v.logs { + s += "

" + vv + "

" + } + w.Write([]byte(s)) +} diff --git a/lib/mux/web_test.go b/lib/mux/web_test.go new file mode 100644 index 0000000..91a0430 --- /dev/null +++ b/lib/mux/web_test.go @@ -0,0 +1,7 @@ +package mux + +import "testing" + +func TestWeb(t *testing.T) { + NewLogServer() +} From d23ed2126dc97eef6648a7839771953bdd615372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=B2=B3?= Date: Mon, 14 Oct 2019 23:46:00 +0800 Subject: [PATCH 23/38] change now time --- lib/mux/web.go | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/lib/mux/web.go b/lib/mux/web.go index 0ebe8e7..d654f15 100644 --- a/lib/mux/web.go +++ b/lib/mux/web.go @@ -17,8 +17,10 @@ type connLog struct { var m map[int]*connLog var copyMap map[int]*connLog +var stashTimeNow time.Time func deepCopyMap() { + stashTimeNow = time.Now() copyMap = make(map[int]*connLog) for k, v := range m { copyMap[k] = &connLog{ @@ -29,24 +31,24 @@ func deepCopyMap() { } } -//func init() { -// m = make(map[int]*connLog) -// m[0] = &connLog{ -// startTime: time.Now(), -// isClose: false, -// logs: []string{"111", "222", "333"}, -// } -// m[1] = &connLog{ -// startTime: time.Now(), -// isClose: false, -// logs: []string{"111", "222", "333", "444"}, -// } -// m[2] = &connLog{ -// startTime: time.Now(), -// isClose: true, -// logs: []string{"111", "222", "333", "555"}, -// } -//} +func init() { + m = make(map[int]*connLog) + m[0] = &connLog{ + startTime: time.Now(), + isClose: false, + logs: []string{"111", "222", "333"}, + } + m[1] = &connLog{ + startTime: time.Now(), + isClose: false, + logs: []string{"111", "222", "333", "444"}, + } + m[2] = &connLog{ + startTime: time.Now(), + isClose: true, + logs: []string{"111", "222", "333", "555"}, + } +} type IntSlice []int @@ -78,7 +80,7 @@ func index(w http.ResponseWriter, r *http.Request) { for v := range keys { connL := copyMap[v] s += "" + strconv.Itoa(v) + "----------" - s += strconv.Itoa(int(time.Now().Unix()-connL.startTime.Unix())) + "s----------" + s += strconv.Itoa(int(stashTimeNow.Unix()-connL.startTime.Unix())) + "s----------" s += strconv.FormatBool(connL.isClose) s += "
" } From c2f4510a0f3c91852092263c16a2540e4236a658 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Tue, 15 Oct 2019 16:32:21 +0800 Subject: [PATCH 24/38] add test code --- lib/mux/conn.go | 26 +++++++++- lib/mux/mux.go | 4 +- lib/mux/mux_test.go | 1 + lib/mux/web.go | 119 ++++++++++++++++++++++++++++++++------------ 4 files changed, 115 insertions(+), 35 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 2d519cb..dc3063c 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -4,6 +4,7 @@ import ( "errors" "io" "net" + "strconv" "sync" "time" @@ -21,9 +22,10 @@ type conn struct { receiveWindow *ReceiveWindow sendWindow *SendWindow once sync.Once + label string } -func NewConn(connId int32, mux *Mux) *conn { +func NewConn(connId int32, mux *Mux, label ...string) *conn { c := &conn{ getStatusCh: make(chan struct{}), connStatusOkCh: make(chan struct{}), @@ -33,8 +35,17 @@ func NewConn(connId int32, mux *Mux) *conn { sendWindow: new(SendWindow), once: sync.Once{}, } + if len(label) > 0 { + c.label = label[0] + } c.receiveWindow.New(mux) c.sendWindow.New(mux) + logm := &connLog{ + startTime: time.Now(), + isClose: false, + logs: []string{c.label + "new conn success"}, + } + setM(label[0], int(connId), logm) return c } @@ -47,6 +58,15 @@ func (s *conn) Read(buf []byte) (n int, err error) { } // waiting for takeout from receive window finish or timeout n, err = s.receiveWindow.Read(buf, s.connId) + var errstr string + if err == nil { + errstr = "err:nil" + } else { + errstr = err.Error() + } + d := getM(s.label, int(s.connId)) + d.logs = append(d.logs, s.label+"read "+strconv.Itoa(n)+" "+errstr) + setM(s.label, int(s.connId), d) return } @@ -81,6 +101,10 @@ func (s *conn) closeProcess() { } s.sendWindow.CloseWindow() s.receiveWindow.CloseWindow() + d := getM(s.label, int(s.connId)) + d.isClose = true + d.logs = append(d.logs, s.label+"close "+time.Now().String()) + setM(s.label, int(s.connId), d) return } diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 42aa8c8..c24c0bc 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -61,7 +61,7 @@ func (s *Mux) NewConn() (*conn, error) { if s.IsClose { return nil, errors.New("the mux has closed") } - conn := NewConn(s.getId(), s) + conn := NewConn(s.getId(), s, "nps ") //it must be set before send s.connMap.Set(conn.connId, conn) s.sendInfo(common.MUX_NEW_CONN, conn.connId, nil) @@ -238,7 +238,7 @@ func (s *Mux) readSession() { s.pingOk = 0 switch pack.Flag { case common.MUX_NEW_CONN: //new connection - connection := NewConn(pack.Id, s) + connection := NewConn(pack.Id, s, "npc ") s.connMap.Set(pack.Id, connection) //it has been set before send ok s.newConnCh <- connection s.sendInfo(common.MUX_NEW_CONN_OK, connection.connId, nil) diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index abc4eb4..1ef71f6 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -111,6 +111,7 @@ func TestNewMux(t *testing.T) { } }() + go NewLogServer() time.Sleep(time.Second * 5) //go test_request() diff --git a/lib/mux/web.go b/lib/mux/web.go index d654f15..36b2017 100644 --- a/lib/mux/web.go +++ b/lib/mux/web.go @@ -2,9 +2,12 @@ package mux import ( "fmt" + "github.com/astaxie/beego/logs" "net/http" "sort" "strconv" + "strings" + "sync" "time" ) @@ -14,16 +17,29 @@ type connLog struct { logs []string } -var m map[int]*connLog +var logms map[int]*connLog +var logmc map[int]*connLog -var copyMap map[int]*connLog +var copyMaps map[int]*connLog +var copyMapc map[int]*connLog var stashTimeNow time.Time +var mutex sync.Mutex -func deepCopyMap() { - stashTimeNow = time.Now() - copyMap = make(map[int]*connLog) - for k, v := range m { - copyMap[k] = &connLog{ +func deepCopyMaps() { + copyMaps = make(map[int]*connLog) + for k, v := range logms { + copyMaps[k] = &connLog{ + startTime: v.startTime, + isClose: v.isClose, + logs: v.logs, + } + } +} + +func deepCopyMapc() { + copyMapc = make(map[int]*connLog) + for k, v := range logmc { + copyMapc[k] = &connLog{ startTime: v.startTime, isClose: v.isClose, logs: v.logs, @@ -32,22 +48,8 @@ func deepCopyMap() { } func init() { - m = make(map[int]*connLog) - m[0] = &connLog{ - startTime: time.Now(), - isClose: false, - logs: []string{"111", "222", "333"}, - } - m[1] = &connLog{ - startTime: time.Now(), - isClose: false, - logs: []string{"111", "222", "333", "444"}, - } - m[2] = &connLog{ - startTime: time.Now(), - isClose: true, - logs: []string{"111", "222", "333", "555"}, - } + logms = make(map[int]*connLog) + logmc = make(map[int]*connLog) } type IntSlice []int @@ -66,21 +68,64 @@ func NewLogServer() { } func stash(w http.ResponseWriter, r *http.Request) { - deepCopyMap() + stashTimeNow = time.Now() + deepCopyMaps() + deepCopyMapc() w.Write([]byte("ok")) } +func getM(label string, id int) (cL *connLog) { + label = strings.TrimSpace(label) + mutex.Lock() + defer mutex.Unlock() + if label == "nps" { + cL = logms[id] + } + if label == "npc" { + cL = logmc[id] + } + return +} + +func setM(label string, id int, cL *connLog) { + label = strings.TrimSpace(label) + mutex.Lock() + defer mutex.Unlock() + if label == "nps" { + logms[id] = cL + } + if label == "npc" { + logmc[id] = cL + } +} + func index(w http.ResponseWriter, r *http.Request) { var keys []int - for k := range copyMap { + for k := range copyMaps { keys = append(keys, k) } sort.Sort(IntSlice(keys)) var s string - for v := range keys { - connL := copyMap[v] - s += "" + strconv.Itoa(v) + "----------" - s += strconv.Itoa(int(stashTimeNow.Unix()-connL.startTime.Unix())) + "s----------" + s += "

nps

" + for _, v := range keys { + connL := copyMaps[v] + s += "" + strconv.Itoa(v) + "----------" + s += strconv.Itoa(int(stashTimeNow.Sub(connL.startTime).Milliseconds())) + "ms----------" + s += strconv.FormatBool(connL.isClose) + s += "
" + } + + keys = keys[:0] + s += "

npc

" + for k := range copyMapc { + keys = append(keys, k) + } + sort.Sort(IntSlice(keys)) + + for _, v := range keys { + connL := copyMapc[v] + s += "" + strconv.Itoa(v) + "----------" + s += strconv.Itoa(int(stashTimeNow.Sub(connL.startTime).Milliseconds())) + "ms----------" s += strconv.FormatBool(connL.isClose) s += "
" } @@ -89,11 +134,21 @@ func index(w http.ResponseWriter, r *http.Request) { func detail(w http.ResponseWriter, r *http.Request) { id := r.FormValue("id") + label := r.FormValue("label") + logs.Warn(label) i, _ := strconv.Atoi(id) - v, _ := copyMap[i] + var v *connLog + if label == "nps" { + v, _ = copyMaps[i] + } + if label == "npc" { + v, _ = copyMapc[i] + } var s string - for _, vv := range v.logs { - s += "

" + vv + "

" + if v != nil { + for i, vv := range v.logs { + s += "

" + strconv.Itoa(i+1) + ":" + vv + "

" + } } w.Write([]byte(s)) } From 23b023c562e582d3c52f1ade00168ab5e900dc0c Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Mon, 21 Oct 2019 11:55:29 +0800 Subject: [PATCH 25/38] add lock free queue --- lib/common/util.go | 2 +- lib/mux/conn.go | 16 +- lib/mux/mux.go | 10 +- lib/mux/mux_test.go | 132 +++++++++++---- lib/mux/queue.go | 390 ++++++++++++++++++++++++++++++++++---------- 5 files changed, 419 insertions(+), 131 deletions(-) diff --git a/lib/common/util.go b/lib/common/util.go index e3dfb4f..1f54a6f 100755 --- a/lib/common/util.go +++ b/lib/common/util.go @@ -263,7 +263,7 @@ func GetPortByAddr(addr string) int { return p } -func CopyBuffer(dst io.Writer, src io.Reader) (written int64, err error) { +func CopyBuffer(dst io.Writer, src io.Reader, label ...string) (written int64, err error) { buf := CopyBuff.Get() defer CopyBuff.Put(buf) for { diff --git a/lib/mux/conn.go b/lib/mux/conn.go index dc3063c..3011732 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -2,10 +2,12 @@ package mux import ( "errors" + "github.com/astaxie/beego/logs" "io" "net" "strconv" "sync" + "sync/atomic" "time" "github.com/cnlh/nps/lib/common" @@ -65,7 +67,7 @@ func (s *conn) Read(buf []byte) (n int, err error) { errstr = err.Error() } d := getM(s.label, int(s.connId)) - d.logs = append(d.logs, s.label+"read "+strconv.Itoa(n)+" "+errstr) + d.logs = append(d.logs, s.label+"read "+strconv.Itoa(n)+" "+errstr+" "+string(buf[:100])) setM(s.label, int(s.connId), d) return } @@ -187,11 +189,7 @@ func (Self *ReceiveWindow) RemainingSize() (n uint32) { func (Self *ReceiveWindow) ReadSize() (n uint32) { // acknowledge the size already read - Self.bufQueue.mutex.Lock() - n = Self.readLength - Self.readLength = 0 - Self.bufQueue.mutex.Unlock() - return + return atomic.SwapUint32(&Self.readLength, 0) } func (Self *ReceiveWindow) CalcSize() { @@ -270,10 +268,8 @@ copyData: //Self.bw.SetCopySize(l) pOff += l Self.off += uint32(l) - Self.bufQueue.mutex.Lock() - Self.readLength += uint32(l) + atomic.AddUint32(&Self.readLength, uint32(l)) //logs.Warn("window read length buf len", Self.readLength, Self.bufQueue.Len()) - Self.bufQueue.mutex.Unlock() n += l l = 0 //Self.bw.EndRead() @@ -422,6 +418,7 @@ func (Self *SendWindow) WriteTo() (p []byte, part bool, err error) { if len(Self.buf[Self.off:]) > common.MAXIMUM_SEGMENT_SIZE { sendSize = common.MAXIMUM_SEGMENT_SIZE part = true + logs.Warn("cut buf by mss") } else { sendSize = uint32(len(Self.buf[Self.off:])) part = false @@ -430,6 +427,7 @@ func (Self *SendWindow) WriteTo() (p []byte, part bool, err error) { // usable window size is small than // window MAXIMUM_SEGMENT_SIZE or send buf left sendSize = Self.RemainingSize() + logs.Warn("cut buf by remainingsize", sendSize, len(Self.buf[Self.off:])) part = true } //logs.Warn("send size", sendSize) diff --git a/lib/mux/mux.go b/lib/mux/mux.go index c24c0bc..6f08641 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -34,6 +34,8 @@ type Mux struct { } func NewMux(c net.Conn, connType string) *Mux { + //c.(*net.TCPConn).SetReadBuffer(0) + //c.(*net.TCPConn).SetWriteBuffer(0) m := &Mux{ conn: c, connMap: NewConnMap(), @@ -173,10 +175,6 @@ func (s *Mux) ping() { select { case <-ticker.C: } - //Avoid going beyond the scope - if (math.MaxInt32 - s.id) < 10000 { - s.id = 0 - } now, _ := time.Now().UTC().MarshalText() s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) if !s.pingTimer.Stop() { @@ -321,6 +319,10 @@ func (s *Mux) Close() error { //get new connId as unique flag func (s *Mux) getId() (id int32) { + //Avoid going beyond the scope + if (math.MaxInt32 - s.id) < 10000 { + atomic.SwapInt32(&s.id, 0) + } id = atomic.AddInt32(&s.id, 1) if _, ok := s.connMap.Get(id); ok { s.getId() diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index 1ef71f6..43e12e8 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -3,13 +3,16 @@ package mux import ( "bufio" "fmt" + "io" "net" "net/http" "net/http/httputil" _ "net/http/pprof" + "strconv" "sync" "testing" "time" + "unsafe" "github.com/astaxie/beego/logs" "github.com/cnlh/nps/lib/common" @@ -30,20 +33,22 @@ func TestNewMux(t *testing.T) { go func() { m2 := NewMux(conn2, "tcp") for { - logs.Warn("npc starting accept") + //logs.Warn("npc starting accept") c, err := m2.Accept() if err != nil { logs.Warn(err) continue } - logs.Warn("npc accept success ") + //logs.Warn("npc accept success ") c2, err := net.Dial("tcp", "127.0.0.1:80") if err != nil { logs.Warn(err) c.Close() continue } - go func(c2 net.Conn, c net.Conn) { + //c2.(*net.TCPConn).SetReadBuffer(0) + //c2.(*net.TCPConn).SetReadBuffer(0) + go func(c2 net.Conn, c *conn) { wg := sync.WaitGroup{} wg.Add(1) go func() { @@ -51,7 +56,7 @@ func TestNewMux(t *testing.T) { if err != nil { c2.Close() c.Close() - logs.Warn("close npc by copy from nps", err) + logs.Warn("close npc by copy from nps", err, c.connId) } wg.Done() }() @@ -61,13 +66,13 @@ func TestNewMux(t *testing.T) { if err != nil { c2.Close() c.Close() - logs.Warn("close npc by copy from server", err) + logs.Warn("close npc by copy from server", err, c.connId) } wg.Done() }() - logs.Warn("npc wait") + //logs.Warn("npc wait") wg.Wait() - }(c2, c) + }(c2, c.(*conn)) } }() @@ -78,42 +83,46 @@ func TestNewMux(t *testing.T) { logs.Warn(err) } for { - logs.Warn("nps starting accept") - conn, err := l.Accept() + //logs.Warn("nps starting accept") + conns, err := l.Accept() if err != nil { logs.Warn(err) continue } - logs.Warn("nps accept success starting new conn") + //conns.(*net.TCPConn).SetReadBuffer(0) + //conns.(*net.TCPConn).SetReadBuffer(0) + //logs.Warn("nps accept success starting new conn") tmpCpnn, err := m1.NewConn() if err != nil { logs.Warn("nps new conn err ", err) continue } logs.Warn("nps new conn success ", tmpCpnn.connId) - go func(tmpCpnn net.Conn, conn net.Conn) { + go func(tmpCpnn *conn, conns net.Conn) { go func() { - _, err := common.CopyBuffer(tmpCpnn, conn) + _, err := common.CopyBuffer(tmpCpnn, conns) if err != nil { - conn.Close() + conns.Close() tmpCpnn.Close() - logs.Warn("close nps by copy from user") + logs.Warn("close nps by copy from user", tmpCpnn.connId, err) } }() //time.Sleep(time.Second) - _, err = common.CopyBuffer(conn, tmpCpnn) + _, err = common.CopyBuffer(conns, tmpCpnn) if err != nil { - conn.Close() + conns.Close() tmpCpnn.Close() - logs.Warn("close nps by copy from npc ") + logs.Warn("close nps by copy from npc ", tmpCpnn.connId, err) } - }(tmpCpnn, conn) + }(tmpCpnn, conns) } }() go NewLogServer() time.Sleep(time.Second * 5) - //go test_request() + //for i:=0;i<1000;i++ { + // go test_raw(i) + //} for { time.Sleep(time.Second * 5) @@ -168,23 +177,40 @@ Connection: keep-alive } } -func test_raw() { - conn, _ := net.Dial("tcp", "127.0.0.1:7777") - for { - conn.Write([]byte(`GET /videojs5/test HTTP/1.1 +func test_raw(k int) { + for i := 0; i < 1; i++ { + ti := time.Now() + conn, _ := net.Dial("tcp", "127.0.0.1:7777") + tid := time.Now() + conn.Write([]byte(`GET / HTTP/1.1 Host: 127.0.0.1:7777 -Connection: keep-alive `)) - buf := make([]byte, 1000000) - n, err := conn.Read(buf) + tiw := time.Now() + buf := make([]byte, 3572) + n, err := io.ReadFull(conn, buf) + //n, err := conn.Read(buf) if err != nil { logs.Warn("close by read response err", err) break } - logs.Warn(n, string(buf[:50]), "\n--------------\n", string(buf[n-50:n])) - time.Sleep(time.Second) + //logs.Warn(n, string(buf[:50]), "\n--------------\n", string(buf[n-50:n])) + //time.Sleep(time.Second) + err = conn.Close() + if err != nil { + logs.Warn("close conn err ", err) + } + now := time.Now() + du := now.Sub(ti).Seconds() + dud := now.Sub(tid).Seconds() + duw := now.Sub(tiw).Seconds() + if du > 1 { + logs.Warn("duration long", du, dud, duw, k, i) + } + if n != 3572 { + logs.Warn("n loss", n, string(buf)) + } } } @@ -199,3 +225,53 @@ func TestNewConn(t *testing.T) { logs.Warn(copy(buf[:3], b), len(buf), cap(buf)) logs.Warn(len(buf), buf[0]) } + +func TestDQueue(t *testing.T) { + logs.EnableFuncCallDepth(true) + logs.SetLogFuncCallDepth(3) + d := new(bufDequeue) + d.vals = make([]unsafe.Pointer, 8) + go func() { + time.Sleep(time.Second) + for i := 0; i < 10; i++ { + logs.Warn(i) + logs.Warn(d.popTail()) + } + }() + go func() { + time.Sleep(time.Second) + for i := 0; i < 10; i++ { + data := "test" + go logs.Warn(i, unsafe.Pointer(&data), d.pushHead(unsafe.Pointer(&data))) + } + }() + time.Sleep(time.Second * 3) +} + +func TestChain(t *testing.T) { + logs.EnableFuncCallDepth(true) + logs.SetLogFuncCallDepth(3) + d := new(bufChain) + d.new(256) + go func() { + time.Sleep(time.Second) + for i := 0; i < 1000; i++ { + unsa, ok := d.popTail() + str := (*string)(unsa) + if ok { + logs.Warn(i, str, *str, ok) + } else { + logs.Warn("nil", i, ok) + } + } + }() + go func() { + time.Sleep(time.Second) + for i := 0; i < 1000; i++ { + data := "test " + strconv.Itoa(i) + logs.Warn(data, unsafe.Pointer(&data)) + go d.pushHead(unsafe.Pointer(&data)) + } + }() + time.Sleep(time.Second * 10) +} diff --git a/lib/mux/queue.go b/lib/mux/queue.go index a835e2a..ef0c904 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -1,19 +1,19 @@ package mux import ( - "container/list" "errors" "github.com/cnlh/nps/lib/common" "io" - "sync" + "math" + "sync/atomic" "time" + "unsafe" ) type QueueOp struct { readOp chan struct{} cleanOp chan struct{} - popWait bool - mutex sync.Mutex + popWait int32 } func (Self *QueueOp) New() { @@ -22,15 +22,15 @@ func (Self *QueueOp) New() { } func (Self *QueueOp) allowPop() (closed bool) { - Self.mutex.Lock() - Self.popWait = false - Self.mutex.Unlock() - select { - case Self.readOp <- struct{}{}: - return false - case <-Self.cleanOp: - return true + if atomic.CompareAndSwapInt32(&Self.popWait, 1, 0) { + select { + case Self.readOp <- struct{}{}: + return false + case <-Self.cleanOp: + return true + } } + return } func (Self *QueueOp) Clean() { @@ -40,84 +40,72 @@ func (Self *QueueOp) Clean() { } type PriorityQueue struct { - list *list.List QueueOp + highestChain *bufChain + middleChain *bufChain + lowestChain *bufChain + hunger uint8 } func (Self *PriorityQueue) New() { - Self.list = list.New() + Self.highestChain = new(bufChain) + Self.highestChain.new(4) + Self.middleChain = new(bufChain) + Self.middleChain.new(32) + Self.lowestChain = new(bufChain) + Self.lowestChain.new(256) Self.QueueOp.New() } func (Self *PriorityQueue) Push(packager *common.MuxPackager) { - Self.mutex.Lock() switch packager.Flag { case common.MUX_PING_FLAG, common.MUX_PING_RETURN: - Self.list.PushFront(packager) + Self.highestChain.pushHead(unsafe.Pointer(packager)) // the ping package need highest priority // prevent ping calculation error - case common.MUX_CONN_CLOSE: - Self.insert(packager) - // the close package may need priority too, set second - // prevent wait too long to close conn + case common.MUX_NEW_CONN, common.MUX_NEW_CONN_OK, common.MUX_NEW_CONN_Fail: + // the new conn package need some priority too + Self.middleChain.pushHead(unsafe.Pointer(packager)) default: - Self.list.PushBack(packager) + Self.lowestChain.pushHead(unsafe.Pointer(packager)) } - if Self.popWait { - Self.mutex.Unlock() - Self.allowPop() - return - } - Self.mutex.Unlock() + Self.allowPop() return } -func (Self *PriorityQueue) insert(packager *common.MuxPackager) { - element := Self.list.Back() - for { - if element == nil { // PriorityQueue dose not have any of msg package with this close package id - element = Self.list.Front() - if element != nil { - Self.list.InsertAfter(packager, element) - // insert close package to second - } else { - Self.list.PushFront(packager) - // list is empty, push to front - } - break - } - if element.Value.(*common.MuxPackager).Flag == common.MUX_NEW_MSG && - element.Value.(*common.MuxPackager).Id == packager.Id { - Self.list.InsertAfter(packager, element) // PriorityQueue has some msg package - // with this close package id, insert close package after last msg package - break - } - element = element.Prev() - } -} - func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { - Self.mutex.Lock() - element := Self.list.Front() - if element != nil { - packager = element.Value.(*common.MuxPackager) - Self.list.Remove(element) - Self.mutex.Unlock() +startPop: + ptr, ok := Self.highestChain.popTail() + if ok { + packager = (*common.MuxPackager)(ptr) return } - Self.popWait = true // PriorityQueue is empty, notice Push method - Self.mutex.Unlock() - select { - case <-Self.readOp: - return Self.Pop() - case <-Self.cleanOp: - return nil + if Self.hunger < 100 { + ptr, ok = Self.middleChain.popTail() + if ok { + packager = (*common.MuxPackager)(ptr) + Self.hunger++ + return + } } -} - -func (Self *PriorityQueue) Len() (n int) { - n = Self.list.Len() - return + ptr, ok = Self.lowestChain.popTail() + if ok { + packager = (*common.MuxPackager)(ptr) + if Self.hunger > 0 { + Self.hunger = uint8(Self.hunger / 2) + } + return + } + // PriorityQueue is empty, notice Push method + if atomic.CompareAndSwapInt32(&Self.popWait, 0, 1) { + select { + case <-Self.readOp: + goto startPop + case <-Self.cleanOp: + return nil + } + } + goto startPop } type ListElement struct { @@ -137,36 +125,36 @@ func (Self *ListElement) New(buf []byte, l uint16, part bool) (err error) { } type FIFOQueue struct { - list []*ListElement + QueueOp + chain *bufChain length uint32 stopOp chan struct{} timeout time.Time - QueueOp } func (Self *FIFOQueue) New() { Self.QueueOp.New() + Self.chain = new(bufChain) + Self.chain.new(64) Self.stopOp = make(chan struct{}, 1) } func (Self *FIFOQueue) Push(element *ListElement) { - Self.mutex.Lock() - Self.list = append(Self.list, element) + Self.chain.pushHead(unsafe.Pointer(element)) Self.length += uint32(element.l) - if Self.popWait { - Self.mutex.Unlock() - Self.allowPop() - return - } - Self.mutex.Unlock() + Self.allowPop() return } func (Self *FIFOQueue) Pop() (element *ListElement, err error) { - Self.mutex.Lock() - if len(Self.list) == 0 { - Self.popWait = true - Self.mutex.Unlock() +startPop: + ptr, ok := Self.chain.popTail() + if ok { + element = (*ListElement)(ptr) + Self.length -= uint32(element.l) + return + } + if atomic.CompareAndSwapInt32(&Self.popWait, 0, 1) { t := Self.timeout.Sub(time.Now()) if t <= 0 { t = time.Minute @@ -175,7 +163,7 @@ func (Self *FIFOQueue) Pop() (element *ListElement, err error) { defer timer.Stop() select { case <-Self.readOp: - Self.mutex.Lock() + goto startPop case <-Self.cleanOp: return case <-Self.stopOp: @@ -186,11 +174,7 @@ func (Self *FIFOQueue) Pop() (element *ListElement, err error) { return } } - element = Self.list[0] - Self.list = Self.list[1:] - Self.length -= uint32(element.l) - Self.mutex.Unlock() - return + goto startPop } func (Self *FIFOQueue) Len() (n uint32) { @@ -204,3 +188,231 @@ func (Self *FIFOQueue) Stop() { func (Self *FIFOQueue) SetTimeOut(t time.Time) { Self.timeout = t } + +// https://golang.org/src/sync/poolqueue.go + +type bufDequeue struct { + // headTail packs together a 32-bit head index and a 32-bit + // tail index. Both are indexes into vals modulo len(vals)-1. + // + // tail = index of oldest data in queue + // head = index of next slot to fill + // + // Slots in the range [tail, head) are owned by consumers. + // A consumer continues to own a slot outside this range until + // it nils the slot, at which point ownership passes to the + // producer. + // + // The head index is stored in the most-significant bits so + // that we can atomically add to it and the overflow is + // harmless. + headTail uint64 + + // vals is a ring buffer of interface{} values stored in this + // dequeue. The size of this must be a power of 2. + // + // A slot is still in use until *both* the tail + // index has moved beyond it and typ has been set to nil. This + // is set to nil atomically by the consumer and read + // atomically by the producer. + vals []unsafe.Pointer +} + +const dequeueBits = 32 + +// dequeueLimit is the maximum size of a bufDequeue. +// +// This must be at most (1<> dequeueBits) & mask) + tail = uint32(ptrs & mask) + return +} + +func (d *bufDequeue) pack(head, tail uint32) uint64 { + const mask = 1<= dequeueLimit { + // Can't make it any bigger. + newSize = dequeueLimit + } + + d2 := &bufChainElt{prev: d} + d2.vals = make([]unsafe.Pointer, newSize) + storePoolChainElt(&c.head, d2) + storePoolChainElt(&d.next, d2) + d2.pushHead(val) + atomic.SwapInt32(&c.chainStatus, 0) + } + } +} + +func (c *bufChain) popTail() (unsafe.Pointer, bool) { + d := loadPoolChainElt(&c.tail) + if d == nil { + return nil, false + } + + for { + // It's important that we load the next pointer + // *before* popping the tail. In general, d may be + // transiently empty, but if next is non-nil before + // the pop and the pop fails, then d is permanently + // empty, which is the only condition under which it's + // safe to drop d from the chain. + d2 := loadPoolChainElt(&d.next) + + if val, ok := d.popTail(); ok { + return val, ok + } + + if d2 == nil { + // This is the only dequeue. It's empty right + // now, but could be pushed to in the future. + return nil, false + } + + // The tail of the chain has been drained, so move on + // to the next dequeue. Try to drop it from the chain + // so the next pop doesn't have to look at the empty + // dequeue again. + if atomic.CompareAndSwapPointer((*unsafe.Pointer)(unsafe.Pointer(&c.tail)), unsafe.Pointer(d), unsafe.Pointer(d2)) { + // We won the race. Clear the prev pointer so + // the garbage collector can collect the empty + // dequeue and so popHead doesn't back up + // further than necessary. + storePoolChainElt(&d2.prev, nil) + } + d = d2 + } +} From 442354db17703919fa51e8c096e36e628708e3ad Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Wed, 23 Oct 2019 23:35:39 +0800 Subject: [PATCH 26/38] add test code --- lib/mux/conn.go | 93 +++++-------- lib/mux/mux.go | 6 +- lib/mux/mux_test.go | 330 ++++++++++++++++++++++++++++++++------------ lib/mux/queue.go | 32 +++-- 4 files changed, 300 insertions(+), 161 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 3011732..99b6a05 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -2,7 +2,6 @@ package mux import ( "errors" - "github.com/astaxie/beego/logs" "io" "net" "strconv" @@ -178,21 +177,17 @@ func (Self *ReceiveWindow) New(mux *Mux) { Self.window.New() } -func (Self *ReceiveWindow) RemainingSize() (n uint32) { +func (Self *ReceiveWindow) remainingSize() (n uint32) { // receive window remaining - if Self.maxSize >= Self.bufQueue.Len() { - n = Self.maxSize - Self.bufQueue.Len() - } - // if maxSize is small than bufQueue length, return 0 - return + return Self.maxSize - Self.bufQueue.Len() } -func (Self *ReceiveWindow) ReadSize() (n uint32) { +func (Self *ReceiveWindow) readSize() (n uint32) { // acknowledge the size already read return atomic.SwapUint32(&Self.readLength, 0) } -func (Self *ReceiveWindow) CalcSize() { +func (Self *ReceiveWindow) calcSize() { // calculating maximum receive window size if Self.count == 0 { //logs.Warn("ping, bw", Self.mux.latency, Self.bw.Get()) @@ -222,22 +217,22 @@ func (Self *ReceiveWindow) Write(buf []byte, l uint16, part bool, id int32) (err if Self.closeOp { return errors.New("conn.receiveWindow: write on closed window") } - element := ListElement{} + element := new(ListElement) err = element.New(buf, l, part) //logs.Warn("push the buf", len(buf), l, (&element).l) if err != nil { return } - Self.bufQueue.Push(&element) // must push data before allow read + Self.bufQueue.Push(element) // must push data before allow read //logs.Warn("read session calc size ", Self.maxSize) // calculating the receive window size - Self.CalcSize() + Self.calcSize() //logs.Warn("read session calc size finish", Self.maxSize) - if Self.RemainingSize() == 0 { + if Self.remainingSize() == 0 { Self.windowFull = true //logs.Warn("window full true", Self.windowFull) } - Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.ReadSize()) + Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.readSize()) return nil } @@ -273,10 +268,10 @@ copyData: n += l l = 0 //Self.bw.EndRead() - Self.sendStatus(id) if Self.off == uint32(Self.element.l) { //logs.Warn("put the element end ", string(Self.element.buf[:15])) common.WindowBuff.Put(Self.element.buf) + Self.sendStatus(id) } if pOff < len(p) && Self.element.part { // element is a part of the segments, trying to fill up buf p @@ -289,7 +284,7 @@ func (Self *ReceiveWindow) sendStatus(id int32) { if Self.windowFull || Self.bufQueue.Len() == 0 { // window is full before read or empty now Self.windowFull = false - Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.ReadSize()) + Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.readSize()) // acknowledge other side, have empty some receive window space //} } @@ -314,11 +309,10 @@ type SendWindow struct { buf []byte sentLength uint32 setSizeCh chan struct{} - setSizeWait bool + setSizeWait int32 unSlide uint32 timeout time.Time window - mutex sync.Mutex } func (Self *SendWindow) New(mux *Mux) { @@ -330,17 +324,12 @@ func (Self *SendWindow) New(mux *Mux) { func (Self *SendWindow) SetSendBuf(buf []byte) { // send window buff from conn write method, set it to send window - Self.mutex.Lock() Self.buf = buf Self.off = 0 - Self.mutex.Unlock() } func (Self *SendWindow) RemainingSize() (n uint32) { - if Self.maxSize >= Self.sentLength { - n = Self.maxSize - Self.sentLength - } - return + return atomic.LoadUint32(&Self.maxSize) - atomic.LoadUint32(&Self.sentLength) } func (Self *SendWindow) SetSize(windowSize, readLength uint32) (closed bool) { @@ -353,25 +342,21 @@ func (Self *SendWindow) SetSize(windowSize, readLength uint32) (closed bool) { close(Self.setSizeCh) return true } - if readLength == 0 && Self.maxSize == windowSize { + + if readLength == 0 && atomic.LoadUint32(&Self.maxSize) == windowSize { //logs.Warn("waiting for another window size") return false // waiting for receive another usable window size } //logs.Warn("set send window size to ", windowSize, readLength) - Self.mutex.Lock() Self.slide(windowSize, readLength) - if Self.setSizeWait { + if Self.RemainingSize() == 0 { + //logs.Warn("waiting for another window size after slide") + // keep the wait status + atomic.StoreInt32(&Self.setSizeWait, 1) + return false + } + if atomic.CompareAndSwapInt32(&Self.setSizeWait, 1, 0) { // send window into the wait status, need notice the channel - //logs.Warn("send window remaining size is 0 , wait") - if Self.RemainingSize() == 0 { - //logs.Warn("waiting for another window size after slide") - // keep the wait status - Self.mutex.Unlock() - return false - } - Self.setSizeWait = false - Self.mutex.Unlock() - //logs.Warn("send window remaining size is 0 starting wait") select { case Self.setSizeCh <- struct{}{}: //logs.Warn("send window remaining size is 0 finish") @@ -382,43 +367,36 @@ func (Self *SendWindow) SetSize(windowSize, readLength uint32) (closed bool) { } } // send window not into the wait status, so just do slide - Self.mutex.Unlock() return false } func (Self *SendWindow) slide(windowSize, readLength uint32) { - Self.sentLength -= readLength - Self.maxSize = windowSize + atomic.AddUint32(&Self.sentLength, ^readLength-1) + atomic.SwapUint32(&Self.maxSize, windowSize) } -func (Self *SendWindow) WriteTo() (p []byte, part bool, err error) { +func (Self *SendWindow) WriteTo() (p []byte, sendSize uint32, part bool, err error) { // returns buf segments, return only one segments, need a loop outside // until err = io.EOF if Self.closeOp { - return nil, false, errors.New("conn.writeWindow: window closed") + return nil, 0, false, errors.New("conn.writeWindow: window closed") } if Self.off == uint32(len(Self.buf)) { - return nil, false, io.EOF + return nil, 0, false, io.EOF // send window buff is drain, return eof and get another one } - Self.mutex.Lock() if Self.RemainingSize() == 0 { - Self.setSizeWait = true - Self.mutex.Unlock() + atomic.StoreInt32(&Self.setSizeWait, 1) // into the wait status err = Self.waitReceiveWindow() if err != nil { - return nil, false, err + return nil, 0, false, err } - } else { - Self.mutex.Unlock() } - Self.mutex.Lock() - var sendSize uint32 if len(Self.buf[Self.off:]) > common.MAXIMUM_SEGMENT_SIZE { sendSize = common.MAXIMUM_SEGMENT_SIZE part = true - logs.Warn("cut buf by mss") + //logs.Warn("cut buf by mss") } else { sendSize = uint32(len(Self.buf[Self.off:])) part = false @@ -427,14 +405,13 @@ func (Self *SendWindow) WriteTo() (p []byte, part bool, err error) { // usable window size is small than // window MAXIMUM_SEGMENT_SIZE or send buf left sendSize = Self.RemainingSize() - logs.Warn("cut buf by remainingsize", sendSize, len(Self.buf[Self.off:])) + //logs.Warn("cut buf by remainingsize", sendSize, len(Self.buf[Self.off:])) part = true } //logs.Warn("send size", sendSize) p = Self.buf[Self.off : sendSize+Self.off] Self.off += sendSize - Self.sentLength += sendSize - Self.mutex.Unlock() + atomic.AddUint32(&Self.sentLength, sendSize) return } @@ -463,8 +440,9 @@ func (Self *SendWindow) WriteFull(buf []byte, id int32) (n int, err error) { Self.SetSendBuf(buf) // set the buf to send window var bufSeg []byte var part bool + var l uint32 for { - bufSeg, part, err = Self.WriteTo() + bufSeg, l, part, err = Self.WriteTo() //logs.Warn("buf seg", len(bufSeg), part, err) // get the buf segments from send window if bufSeg == nil && part == false && err == io.EOF { @@ -475,7 +453,8 @@ func (Self *SendWindow) WriteFull(buf []byte, id int32) (n int, err error) { if err != nil { break } - n += len(bufSeg) + n += int(l) + l = 0 if part { Self.mux.sendInfo(common.MUX_NEW_MSG_PART, id, bufSeg) } else { diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 6f08641..8300977 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -41,7 +41,7 @@ func NewMux(c net.Conn, connType string) *Mux { connMap: NewConnMap(), id: 0, closeChan: make(chan struct{}, 3), - newConnCh: make(chan *conn), + newConnCh: make(chan *conn, 10), bw: new(bandwidth), IsClose: false, connType: connType, @@ -321,11 +321,11 @@ func (s *Mux) Close() error { func (s *Mux) getId() (id int32) { //Avoid going beyond the scope if (math.MaxInt32 - s.id) < 10000 { - atomic.SwapInt32(&s.id, 0) + atomic.StoreInt32(&s.id, 0) } id = atomic.AddInt32(&s.id, 1) if _, ok := s.connMap.Get(id); ok { - s.getId() + return s.getId() } return } diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index 43e12e8..0ac54d5 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -3,19 +3,18 @@ package mux import ( "bufio" "fmt" - "io" + "github.com/cnlh/nps/lib/common" + "log" "net" "net/http" "net/http/httputil" _ "net/http/pprof" "strconv" - "sync" "testing" "time" "unsafe" "github.com/astaxie/beego/logs" - "github.com/cnlh/nps/lib/common" ) var conn1 net.Conn @@ -49,42 +48,42 @@ func TestNewMux(t *testing.T) { //c2.(*net.TCPConn).SetReadBuffer(0) //c2.(*net.TCPConn).SetReadBuffer(0) go func(c2 net.Conn, c *conn) { - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - _, err = common.CopyBuffer(c2, c) - if err != nil { - c2.Close() - c.Close() - logs.Warn("close npc by copy from nps", err, c.connId) - } - wg.Done() - }() - wg.Add(1) - go func() { - _, err = common.CopyBuffer(c, c2) - if err != nil { - c2.Close() - c.Close() - logs.Warn("close npc by copy from server", err, c.connId) - } - wg.Done() - }() - //logs.Warn("npc wait") - wg.Wait() + //wg := sync.WaitGroup{} + //wg.Add(1) + //go func() { + // _, err = common.CopyBuffer(c2, c) + // if err != nil { + // c2.Close() + // c.Close() + // logs.Warn("close npc by copy from nps", err, c.connId) + // } + // wg.Done() + //}() + //wg.Add(1) + //go func() { + // _, err = common.CopyBuffer(c, c2) + // if err != nil { + // c2.Close() + // c.Close() + // logs.Warn("close npc by copy from server", err, c.connId) + // } + // wg.Done() + //}() + ////logs.Warn("npc wait") + //wg.Wait() }(c2, c.(*conn)) } }() go func() { - m1 := NewMux(conn1, "tcp") + //m1 := NewMux(conn1, "tcp") l, err := net.Listen("tcp", "127.0.0.1:7777") if err != nil { logs.Warn(err) } for { //logs.Warn("nps starting accept") - conns, err := l.Accept() + _, err := l.Accept() if err != nil { logs.Warn(err) continue @@ -92,37 +91,37 @@ func TestNewMux(t *testing.T) { //conns.(*net.TCPConn).SetReadBuffer(0) //conns.(*net.TCPConn).SetReadBuffer(0) //logs.Warn("nps accept success starting new conn") - tmpCpnn, err := m1.NewConn() - if err != nil { - logs.Warn("nps new conn err ", err) - continue - } - logs.Warn("nps new conn success ", tmpCpnn.connId) - go func(tmpCpnn *conn, conns net.Conn) { - go func() { - _, err := common.CopyBuffer(tmpCpnn, conns) - if err != nil { - conns.Close() - tmpCpnn.Close() - logs.Warn("close nps by copy from user", tmpCpnn.connId, err) - } - }() - //time.Sleep(time.Second) - _, err = common.CopyBuffer(conns, tmpCpnn) - if err != nil { - conns.Close() - tmpCpnn.Close() - logs.Warn("close nps by copy from npc ", tmpCpnn.connId, err) - } - }(tmpCpnn, conns) + //tmpCpnn, err := m1.NewConn() + //if err != nil { + // logs.Warn("nps new conn err ", err) + // continue + //} + ////logs.Warn("nps new conn success ", tmpCpnn.connId) + //go func(tmpCpnn *conn, conns net.Conn) { + // //go func() { + // // _, err := common.CopyBuffer(tmpCpnn, conns) + // // if err != nil { + // // conns.Close() + // // tmpCpnn.Close() + // // logs.Warn("close nps by copy from user", tmpCpnn.connId, err) + // // } + // //}() + // ////time.Sleep(time.Second) + // //_, err = common.CopyBuffer(conns, tmpCpnn) + // //if err != nil { + // // conns.Close() + // // tmpCpnn.Close() + // // logs.Warn("close nps by copy from npc ", tmpCpnn.connId, err) + // //} + //}(tmpCpnn, conns) } }() go NewLogServer() time.Sleep(time.Second * 5) - //for i:=0;i<1000;i++ { - // go test_raw(i) - //} + for i := 0; i < 1000; i++ { + go test_raw(i) + } for { time.Sleep(time.Second * 5) @@ -180,37 +179,37 @@ Connection: keep-alive func test_raw(k int) { for i := 0; i < 1; i++ { ti := time.Now() - conn, _ := net.Dial("tcp", "127.0.0.1:7777") + _, _ = net.Dial("tcp", "127.0.0.1:7777") tid := time.Now() - conn.Write([]byte(`GET / HTTP/1.1 -Host: 127.0.0.1:7777 - - -`)) - tiw := time.Now() - buf := make([]byte, 3572) - n, err := io.ReadFull(conn, buf) - //n, err := conn.Read(buf) - if err != nil { - logs.Warn("close by read response err", err) - break - } - //logs.Warn(n, string(buf[:50]), "\n--------------\n", string(buf[n-50:n])) - //time.Sleep(time.Second) - err = conn.Close() - if err != nil { - logs.Warn("close conn err ", err) - } + // conn.Write([]byte(`GET / HTTP/1.1 + //Host: 127.0.0.1:7777 + // + // + //`)) + // tiw := time.Now() + //buf := make([]byte, 3572) + //n, err := io.ReadFull(conn, buf) + ////n, err := conn.Read(buf) + //if err != nil { + // logs.Warn("close by read response err", err) + // break + //} + ////logs.Warn(n, string(buf[:50]), "\n--------------\n", string(buf[n-50:n])) + ////time.Sleep(time.Second) + //err = conn.Close() + //if err != nil { + // logs.Warn("close conn err ", err) + //} now := time.Now() du := now.Sub(ti).Seconds() dud := now.Sub(tid).Seconds() - duw := now.Sub(tiw).Seconds() - if du > 1 { - logs.Warn("duration long", du, dud, duw, k, i) - } - if n != 3572 { - logs.Warn("n loss", n, string(buf)) - } + //duw := now.Sub(tiw).Seconds() + //if du > 1 { + logs.Warn("duration long", du, dud, k, i) + //} + //if n != 3572 { + // logs.Warn("n loss", n, string(buf)) + //} } } @@ -249,29 +248,182 @@ func TestDQueue(t *testing.T) { } func TestChain(t *testing.T) { + go func() { + log.Println(http.ListenAndServe("0.0.0.0:8889", nil)) + }() logs.EnableFuncCallDepth(true) logs.SetLogFuncCallDepth(3) + time.Sleep(time.Second * 5) d := new(bufChain) d.new(256) go func() { time.Sleep(time.Second) - for i := 0; i < 1000; i++ { + for i := 0; i < 30000; i++ { unsa, ok := d.popTail() str := (*string)(unsa) if ok { - logs.Warn(i, str, *str, ok) + fmt.Println(i, str, *str, ok) + //logs.Warn(i, str, *str, ok) } else { - logs.Warn("nil", i, ok) + fmt.Println("nil", i, ok) + //logs.Warn("nil", i, ok) } } }() go func() { time.Sleep(time.Second) - for i := 0; i < 1000; i++ { - data := "test " + strconv.Itoa(i) - logs.Warn(data, unsafe.Pointer(&data)) - go d.pushHead(unsafe.Pointer(&data)) + for i := 0; i < 3000; i++ { + go func(i int) { + for n := 0; n < 10; n++ { + data := "test " + strconv.Itoa(i) + strconv.Itoa(n) + fmt.Println(data, unsafe.Pointer(&data)) + //logs.Warn(data, unsafe.Pointer(&data)) + d.pushHead(unsafe.Pointer(&data)) + } + }(i) } }() - time.Sleep(time.Second * 10) + time.Sleep(time.Second * 100000) } + +func TestFIFO(t *testing.T) { + go func() { + log.Println(http.ListenAndServe("0.0.0.0:8889", nil)) + }() + logs.EnableFuncCallDepth(true) + logs.SetLogFuncCallDepth(3) + time.Sleep(time.Second * 5) + d := new(FIFOQueue) + d.New() + go func() { + time.Sleep(time.Second) + for i := 0; i < 30000; i++ { + data, err := d.Pop() + if err == nil { + //fmt.Println(i, string(data.buf), err) + logs.Warn(i, string(data.buf), err) + } else { + //fmt.Println("err", err) + logs.Warn("err", err) + } + } + }() + go func() { + time.Sleep(time.Second * 10) + for i := 0; i < 3000; i++ { + go func(i int) { + for n := 0; n < 10; n++ { + data := new(ListElement) + by := []byte("test " + strconv.Itoa(i) + strconv.Itoa(n)) + _ = data.New(by, uint16(len(by)), true) + //fmt.Println(string((*data).buf), data) + logs.Warn(string((*data).buf), data) + d.Push(data) + } + }(i) + } + }() + time.Sleep(time.Second * 100000) +} + +func TestPriority(t *testing.T) { + go func() { + log.Println(http.ListenAndServe("0.0.0.0:8889", nil)) + }() + logs.EnableFuncCallDepth(true) + logs.SetLogFuncCallDepth(3) + time.Sleep(time.Second * 5) + d := new(PriorityQueue) + d.New() + go func() { + time.Sleep(time.Second) + for i := 0; i < 36000; i++ { + data := d.Pop() + //fmt.Println(i, string(data.buf), err) + logs.Warn(i, string(data.Content), data) + } + }() + go func() { + time.Sleep(time.Second * 10) + for i := 0; i < 3000; i++ { + go func(i int) { + for n := 0; n < 10; n++ { + data := new(common.MuxPackager) + by := []byte("test " + strconv.Itoa(i) + strconv.Itoa(n)) + _ = data.NewPac(common.MUX_NEW_MSG_PART, int32(i), by) + //fmt.Println(string((*data).buf), data) + logs.Warn(string((*data).Content), data) + d.Push(data) + } + }(i) + go func(i int) { + data := new(common.MuxPackager) + _ = data.NewPac(common.MUX_NEW_CONN, int32(i), nil) + //fmt.Println(string((*data).buf), data) + logs.Warn(data) + d.Push(data) + }(i) + go func(i int) { + data := new(common.MuxPackager) + _ = data.NewPac(common.MUX_NEW_CONN_OK, int32(i), nil) + //fmt.Println(string((*data).buf), data) + logs.Warn(data) + d.Push(data) + }(i) + } + }() + time.Sleep(time.Second * 100000) +} + +//func TestReceive(t *testing.T) { +// go func() { +// log.Println(http.ListenAndServe("0.0.0.0:8889", nil)) +// }() +// logs.EnableFuncCallDepth(true) +// logs.SetLogFuncCallDepth(3) +// time.Sleep(time.Second * 5) +// mux := new(Mux) +// mux.bw.readBandwidth = float64(1*1024*1024) +// mux.latency = float64(1/1000) +// wind := new(ReceiveWindow) +// wind.New(mux) +// wind. +// go func() { +// time.Sleep(time.Second) +// for i := 0; i < 36000; i++ { +// data := d.Pop() +// //fmt.Println(i, string(data.buf), err) +// logs.Warn(i, string(data.Content), data) +// } +// }() +// go func() { +// time.Sleep(time.Second*10) +// for i := 0; i < 3000; i++ { +// go func(i int) { +// for n := 0; n < 10; n++{ +// data := new(common.MuxPackager) +// by := []byte("test " + strconv.Itoa(i) + strconv.Itoa(n)) +// _ = data.NewPac(common.MUX_NEW_MSG_PART, int32(i), by) +// //fmt.Println(string((*data).buf), data) +// logs.Warn(string((*data).Content), data) +// d.Push(data) +// } +// }(i) +// go func(i int) { +// data := new(common.MuxPackager) +// _ = data.NewPac(common.MUX_NEW_CONN, int32(i), nil) +// //fmt.Println(string((*data).buf), data) +// logs.Warn(data) +// d.Push(data) +// }(i) +// go func(i int) { +// data := new(common.MuxPackager) +// _ = data.NewPac(common.MUX_NEW_CONN_OK, int32(i), nil) +// //fmt.Println(string((*data).buf), data) +// logs.Warn(data) +// d.Push(data) +// }(i) +// } +// }() +// time.Sleep(time.Second * 100000) +//} diff --git a/lib/mux/queue.go b/lib/mux/queue.go index ef0c904..488c616 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -73,6 +73,8 @@ func (Self *PriorityQueue) Push(packager *common.MuxPackager) { return } +const maxHunger uint8 = 10 + func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { startPop: ptr, ok := Self.highestChain.popTail() @@ -80,7 +82,7 @@ startPop: packager = (*common.MuxPackager)(ptr) return } - if Self.hunger < 100 { + if Self.hunger < maxHunger { ptr, ok = Self.middleChain.popTail() if ok { packager = (*common.MuxPackager)(ptr) @@ -96,6 +98,13 @@ startPop: } return } + if Self.hunger > 0 { + ptr, ok = Self.middleChain.popTail() + if ok { + packager = (*common.MuxPackager)(ptr) + return + } + } // PriorityQueue is empty, notice Push method if atomic.CompareAndSwapInt32(&Self.popWait, 0, 1) { select { @@ -141,7 +150,7 @@ func (Self *FIFOQueue) New() { func (Self *FIFOQueue) Push(element *ListElement) { Self.chain.pushHead(unsafe.Pointer(element)) - Self.length += uint32(element.l) + atomic.AddUint32(&Self.length, uint32(element.l)) Self.allowPop() return } @@ -151,7 +160,7 @@ startPop: ptr, ok := Self.chain.popTail() if ok { element = (*ListElement)(ptr) - Self.length -= uint32(element.l) + atomic.AddUint32(&Self.length, ^uint32(element.l-1)) return } if atomic.CompareAndSwapInt32(&Self.popWait, 0, 1) { @@ -178,7 +187,7 @@ startPop: } func (Self *FIFOQueue) Len() (n uint32) { - return Self.length + return atomic.LoadUint32(&Self.length) } func (Self *FIFOQueue) Stop() { @@ -273,17 +282,16 @@ func (d *bufDequeue) popTail() (unsafe.Pointer, bool) { return nil, false } slot := &d.vals[tail&uint32(len(d.vals)-1)] + var val unsafe.Pointer for { - typ := atomic.LoadPointer(slot) - if typ != nil { + val = atomic.LoadPointer(slot) + if val != nil { + // We now own slot. break } // Another goroutine is still pushing data on the tail. } - // We now own slot. - val := *slot - // Tell pushHead that we're done with this slot. Zeroing the // slot is also important so we don't leave behind references // that could keep this object live longer than necessary. @@ -369,10 +377,10 @@ func (c *bufChain) pushHead(val unsafe.Pointer) { d2 := &bufChainElt{prev: d} d2.vals = make([]unsafe.Pointer, newSize) - storePoolChainElt(&c.head, d2) - storePoolChainElt(&d.next, d2) d2.pushHead(val) - atomic.SwapInt32(&c.chainStatus, 0) + storePoolChainElt(&d.next, d2) + storePoolChainElt(&c.head, d2) + atomic.StoreInt32(&c.chainStatus, 0) } } } From 5f354158496443e4294c2b104d6dc344236fdcba Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sun, 27 Oct 2019 23:25:12 +0800 Subject: [PATCH 27/38] fix queue bug --- lib/mux/conn.go | 77 +++++++++---------- lib/mux/mux.go | 65 ++++++++-------- lib/mux/mux_test.go | 175 +++++++++++++++++++++++--------------------- lib/mux/queue.go | 152 +++++++++++++++++++++++++++----------- 4 files changed, 275 insertions(+), 194 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 99b6a05..f3217d8 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -4,7 +4,6 @@ import ( "errors" "io" "net" - "strconv" "sync" "sync/atomic" "time" @@ -41,12 +40,12 @@ func NewConn(connId int32, mux *Mux, label ...string) *conn { } c.receiveWindow.New(mux) c.sendWindow.New(mux) - logm := &connLog{ - startTime: time.Now(), - isClose: false, - logs: []string{c.label + "new conn success"}, - } - setM(label[0], int(connId), logm) + //logm := &connLog{ + // startTime: time.Now(), + // isClose: false, + // logs: []string{c.label + "new conn success"}, + //} + //setM(label[0], int(connId), logm) return c } @@ -59,15 +58,15 @@ func (s *conn) Read(buf []byte) (n int, err error) { } // waiting for takeout from receive window finish or timeout n, err = s.receiveWindow.Read(buf, s.connId) - var errstr string - if err == nil { - errstr = "err:nil" - } else { - errstr = err.Error() - } - d := getM(s.label, int(s.connId)) - d.logs = append(d.logs, s.label+"read "+strconv.Itoa(n)+" "+errstr+" "+string(buf[:100])) - setM(s.label, int(s.connId), d) + //var errstr string + //if err == nil { + // errstr = "err:nil" + //} else { + // errstr = err.Error() + //} + //d := getM(s.label, int(s.connId)) + //d.logs = append(d.logs, s.label+"read "+strconv.Itoa(n)+" "+errstr+" "+string(buf[:100])) + //setM(s.label, int(s.connId), d) return } @@ -102,10 +101,10 @@ func (s *conn) closeProcess() { } s.sendWindow.CloseWindow() s.receiveWindow.CloseWindow() - d := getM(s.label, int(s.connId)) - d.isClose = true - d.logs = append(d.logs, s.label+"close "+time.Now().String()) - setM(s.label, int(s.connId), d) + //d := getM(s.label, int(s.connId)) + //d.isClose = true + //d.logs = append(d.logs, s.label+"close "+time.Now().String()) + //setM(s.label, int(s.connId), d) return } @@ -154,12 +153,12 @@ func (Self *window) CloseWindow() { } type ReceiveWindow struct { - bufQueue FIFOQueue + bufQueue ReceiveWindowQueue element *ListElement readLength uint32 readOp chan struct{} readWait bool - windowFull bool + windowFull uint32 count int8 //bw *bandwidth once sync.Once @@ -179,7 +178,7 @@ func (Self *ReceiveWindow) New(mux *Mux) { func (Self *ReceiveWindow) remainingSize() (n uint32) { // receive window remaining - return Self.maxSize - Self.bufQueue.Len() + return atomic.LoadUint32(&Self.maxSize) - Self.bufQueue.Len() } func (Self *ReceiveWindow) readSize() (n uint32) { @@ -207,7 +206,7 @@ func (Self *ReceiveWindow) calcSize() { } // set the maximum size //logs.Warn("n", n) - Self.maxSize = n + atomic.StoreUint32(&Self.maxSize, n) Self.count = -10 } Self.count += 1 @@ -229,7 +228,7 @@ func (Self *ReceiveWindow) Write(buf []byte, l uint16, part bool, id int32) (err Self.calcSize() //logs.Warn("read session calc size finish", Self.maxSize) if Self.remainingSize() == 0 { - Self.windowFull = true + atomic.StoreUint32(&Self.windowFull, 1) //logs.Warn("window full true", Self.windowFull) } Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.readSize()) @@ -259,7 +258,7 @@ copyData: } //logs.Warn("pop element", Self.element.l, Self.element.part) } - l = copy(p[pOff:], Self.element.buf[Self.off:]) + l = copy(p[pOff:], Self.element.buf[Self.off:Self.element.l]) //Self.bw.SetCopySize(l) pOff += l Self.off += uint32(l) @@ -281,13 +280,16 @@ copyData: } func (Self *ReceiveWindow) sendStatus(id int32) { - if Self.windowFull || Self.bufQueue.Len() == 0 { + if Self.bufQueue.Len() == 0 { // window is full before read or empty now - Self.windowFull = false - Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.readSize()) + Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, atomic.LoadUint32(&Self.maxSize), Self.readSize()) // acknowledge other side, have empty some receive window space //} } + if atomic.LoadUint32(&Self.windowFull) > 0 && Self.remainingSize() > 0 { + atomic.StoreUint32(&Self.windowFull, 0) + Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, atomic.LoadUint32(&Self.maxSize), Self.readSize()) + } } func (Self *ReceiveWindow) SetTimeOut(t time.Time) { @@ -309,8 +311,7 @@ type SendWindow struct { buf []byte sentLength uint32 setSizeCh chan struct{} - setSizeWait int32 - unSlide uint32 + setSizeWait uint32 timeout time.Time window } @@ -352,10 +353,10 @@ func (Self *SendWindow) SetSize(windowSize, readLength uint32) (closed bool) { if Self.RemainingSize() == 0 { //logs.Warn("waiting for another window size after slide") // keep the wait status - atomic.StoreInt32(&Self.setSizeWait, 1) + //atomic.StoreUint32(&Self.setSizeWait, 1) return false } - if atomic.CompareAndSwapInt32(&Self.setSizeWait, 1, 0) { + if atomic.CompareAndSwapUint32(&Self.setSizeWait, 1, 0) { // send window into the wait status, need notice the channel select { case Self.setSizeCh <- struct{}{}: @@ -372,7 +373,7 @@ func (Self *SendWindow) SetSize(windowSize, readLength uint32) (closed bool) { func (Self *SendWindow) slide(windowSize, readLength uint32) { atomic.AddUint32(&Self.sentLength, ^readLength-1) - atomic.SwapUint32(&Self.maxSize, windowSize) + atomic.StoreUint32(&Self.maxSize, windowSize) } func (Self *SendWindow) WriteTo() (p []byte, sendSize uint32, part bool, err error) { @@ -386,7 +387,7 @@ func (Self *SendWindow) WriteTo() (p []byte, sendSize uint32, part bool, err err // send window buff is drain, return eof and get another one } if Self.RemainingSize() == 0 { - atomic.StoreInt32(&Self.setSizeWait, 1) + atomic.StoreUint32(&Self.setSizeWait, 1) // into the wait status err = Self.waitReceiveWindow() if err != nil { @@ -395,20 +396,20 @@ func (Self *SendWindow) WriteTo() (p []byte, sendSize uint32, part bool, err err } if len(Self.buf[Self.off:]) > common.MAXIMUM_SEGMENT_SIZE { sendSize = common.MAXIMUM_SEGMENT_SIZE - part = true //logs.Warn("cut buf by mss") } else { sendSize = uint32(len(Self.buf[Self.off:])) - part = false } if Self.RemainingSize() < sendSize { // usable window size is small than // window MAXIMUM_SEGMENT_SIZE or send buf left sendSize = Self.RemainingSize() //logs.Warn("cut buf by remainingsize", sendSize, len(Self.buf[Self.off:])) - part = true } //logs.Warn("send size", sendSize) + if sendSize < uint32(len(Self.buf[Self.off:])) { + part = true + } p = Self.buf[Self.off : sendSize+Self.off] Self.off += sendSize atomic.AddUint32(&Self.sentLength, sendSize) diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 8300977..8a6086c 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -1,7 +1,6 @@ package mux import ( - "bytes" "errors" "io" "math" @@ -29,7 +28,7 @@ type Mux struct { pingTimer *time.Timer connType string writeQueue PriorityQueue - bufCh chan *bytes.Buffer + //bufQueue BytesQueue sync.Mutex } @@ -40,16 +39,16 @@ func NewMux(c net.Conn, connType string) *Mux { conn: c, connMap: NewConnMap(), id: 0, - closeChan: make(chan struct{}, 3), + closeChan: make(chan struct{}, 1), newConnCh: make(chan *conn, 10), bw: new(bandwidth), IsClose: false, connType: connType, - bufCh: make(chan *bytes.Buffer), pingCh: make(chan []byte), pingTimer: time.NewTimer(15 * time.Second), } m.writeQueue.New() + //m.bufQueue.New() //read session by flag m.readSession() //ping @@ -111,16 +110,18 @@ func (s *Mux) sendInfo(flag uint8, id int32, data ...interface{}) { func (s *Mux) writeSession() { go s.packBuf() - go s.writeBuf() + //go s.writeBuf() } func (s *Mux) packBuf() { + buffer := common.BuffPool.Get() for { if s.IsClose { break } + buffer.Reset() pack := s.writeQueue.Pop() - buffer := common.BuffPool.Get() + //buffer := common.BuffPool.Get() err := pack.Pack(buffer) common.MuxPack.Put(pack) if err != nil { @@ -129,34 +130,37 @@ func (s *Mux) packBuf() { break } //logs.Warn(buffer.String()) - select { - case s.bufCh <- buffer: - case <-s.closeChan: + //s.bufQueue.Push(buffer) + l := buffer.Len() + n, err := buffer.WriteTo(s.conn) + //common.BuffPool.Put(buffer) + if err != nil || int(n) != l { + logs.Warn("close from write session fail ", err, n, l) + s.Close() break } } } -func (s *Mux) writeBuf() { - for { - if s.IsClose { - break - } - select { - case buffer := <-s.bufCh: - l := buffer.Len() - n, err := buffer.WriteTo(s.conn) - common.BuffPool.Put(buffer) - if err != nil || int(n) != l { - logs.Warn("close from write session fail ", err, n, l) - s.Close() - break - } - case <-s.closeChan: - break - } - } -} +//func (s *Mux) writeBuf() { +// for { +// if s.IsClose { +// break +// } +// buffer, err := s.bufQueue.Pop() +// if err != nil { +// break +// } +// l := buffer.Len() +// n, err := buffer.WriteTo(s.conn) +// common.BuffPool.Put(buffer) +// if err != nil || int(n) != l { +// logs.Warn("close from write session fail ", err, n, l) +// s.Close() +// break +// } +// } +//} func (s *Mux) ping() { go func() { @@ -310,8 +314,7 @@ func (s *Mux) Close() error { } s.IsClose = true s.connMap.Close() - s.closeChan <- struct{}{} - s.closeChan <- struct{}{} + //s.bufQueue.Stop() s.closeChan <- struct{}{} close(s.newConnCh) return s.conn.Close() diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index 0ac54d5..e3f9dcc 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -4,12 +4,14 @@ import ( "bufio" "fmt" "github.com/cnlh/nps/lib/common" + "io" "log" "net" "net/http" "net/http/httputil" _ "net/http/pprof" "strconv" + "sync" "testing" "time" "unsafe" @@ -48,42 +50,42 @@ func TestNewMux(t *testing.T) { //c2.(*net.TCPConn).SetReadBuffer(0) //c2.(*net.TCPConn).SetReadBuffer(0) go func(c2 net.Conn, c *conn) { - //wg := sync.WaitGroup{} - //wg.Add(1) - //go func() { - // _, err = common.CopyBuffer(c2, c) - // if err != nil { - // c2.Close() - // c.Close() - // logs.Warn("close npc by copy from nps", err, c.connId) - // } - // wg.Done() - //}() - //wg.Add(1) - //go func() { - // _, err = common.CopyBuffer(c, c2) - // if err != nil { - // c2.Close() - // c.Close() - // logs.Warn("close npc by copy from server", err, c.connId) - // } - // wg.Done() - //}() - ////logs.Warn("npc wait") - //wg.Wait() + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + _, err = common.CopyBuffer(c2, c) + if err != nil { + c2.Close() + c.Close() + //logs.Warn("close npc by copy from nps", err, c.connId) + } + wg.Done() + }() + wg.Add(1) + go func() { + _, err = common.CopyBuffer(c, c2) + if err != nil { + c2.Close() + c.Close() + //logs.Warn("close npc by copy from server", err, c.connId) + } + wg.Done() + }() + //logs.Warn("npc wait") + wg.Wait() }(c2, c.(*conn)) } }() go func() { - //m1 := NewMux(conn1, "tcp") + m1 := NewMux(conn1, "tcp") l, err := net.Listen("tcp", "127.0.0.1:7777") if err != nil { logs.Warn(err) } for { //logs.Warn("nps starting accept") - _, err := l.Accept() + conns, err := l.Accept() if err != nil { logs.Warn(err) continue @@ -91,37 +93,38 @@ func TestNewMux(t *testing.T) { //conns.(*net.TCPConn).SetReadBuffer(0) //conns.(*net.TCPConn).SetReadBuffer(0) //logs.Warn("nps accept success starting new conn") - //tmpCpnn, err := m1.NewConn() - //if err != nil { - // logs.Warn("nps new conn err ", err) - // continue - //} - ////logs.Warn("nps new conn success ", tmpCpnn.connId) - //go func(tmpCpnn *conn, conns net.Conn) { - // //go func() { - // // _, err := common.CopyBuffer(tmpCpnn, conns) - // // if err != nil { - // // conns.Close() - // // tmpCpnn.Close() - // // logs.Warn("close nps by copy from user", tmpCpnn.connId, err) - // // } - // //}() - // ////time.Sleep(time.Second) - // //_, err = common.CopyBuffer(conns, tmpCpnn) - // //if err != nil { - // // conns.Close() - // // tmpCpnn.Close() - // // logs.Warn("close nps by copy from npc ", tmpCpnn.connId, err) - // //} - //}(tmpCpnn, conns) + tmpCpnn, err := m1.NewConn() + if err != nil { + logs.Warn("nps new conn err ", err) + continue + } + //logs.Warn("nps new conn success ", tmpCpnn.connId) + go func(tmpCpnn *conn, conns net.Conn) { + go func() { + _, err := common.CopyBuffer(tmpCpnn, conns) + if err != nil { + conns.Close() + tmpCpnn.Close() + //logs.Warn("close nps by copy from user", tmpCpnn.connId, err) + } + }() + //time.Sleep(time.Second) + _, err = common.CopyBuffer(conns, tmpCpnn) + if err != nil { + conns.Close() + tmpCpnn.Close() + //logs.Warn("close nps by copy from npc ", tmpCpnn.connId, err) + } + }(tmpCpnn, conns) } }() - go NewLogServer() + //go NewLogServer() time.Sleep(time.Second * 5) for i := 0; i < 1000; i++ { go test_raw(i) } + //test_request() for { time.Sleep(time.Second * 5) @@ -154,7 +157,7 @@ func client() { func test_request() { conn, _ := net.Dial("tcp", "127.0.0.1:7777") for { - conn.Write([]byte(`GET /videojs5/video.js HTTP/1.1 + conn.Write([]byte(`GET / HTTP/1.1 Host: 127.0.0.1:7777 Connection: keep-alive @@ -177,39 +180,42 @@ Connection: keep-alive } func test_raw(k int) { - for i := 0; i < 1; i++ { + for i := 0; i < 10; i++ { ti := time.Now() - _, _ = net.Dial("tcp", "127.0.0.1:7777") + conn, err := net.Dial("tcp", "127.0.0.1:7777") + if err != nil { + logs.Warn("conn dial err", err) + } tid := time.Now() - // conn.Write([]byte(`GET / HTTP/1.1 - //Host: 127.0.0.1:7777 - // - // - //`)) - // tiw := time.Now() - //buf := make([]byte, 3572) - //n, err := io.ReadFull(conn, buf) - ////n, err := conn.Read(buf) - //if err != nil { - // logs.Warn("close by read response err", err) - // break - //} - ////logs.Warn(n, string(buf[:50]), "\n--------------\n", string(buf[n-50:n])) - ////time.Sleep(time.Second) - //err = conn.Close() - //if err != nil { - // logs.Warn("close conn err ", err) - //} + conn.Write([]byte(`GET / HTTP/1.1 +Host: 127.0.0.1:7777 + + +`)) + tiw := time.Now() + buf := make([]byte, 3572) + n, err := io.ReadFull(conn, buf) + //n, err := conn.Read(buf) + if err != nil { + logs.Warn("close by read response err", err) + break + } + logs.Warn(n, string(buf[:50]), "\n--------------\n", string(buf[n-50:n])) + //time.Sleep(time.Second) + err = conn.Close() + if err != nil { + logs.Warn("close conn err ", err) + } now := time.Now() du := now.Sub(ti).Seconds() dud := now.Sub(tid).Seconds() - //duw := now.Sub(tiw).Seconds() - //if du > 1 { - logs.Warn("duration long", du, dud, k, i) - //} - //if n != 3572 { - // logs.Warn("n loss", n, string(buf)) - //} + duw := now.Sub(tiw).Seconds() + if du > 1 { + logs.Warn("duration long", du, dud, duw, k, i) + } + if n != 3572 { + logs.Warn("n loss", n, string(buf)) + } } } @@ -293,11 +299,11 @@ func TestFIFO(t *testing.T) { logs.EnableFuncCallDepth(true) logs.SetLogFuncCallDepth(3) time.Sleep(time.Second * 5) - d := new(FIFOQueue) + d := new(ReceiveWindowQueue) d.New() go func() { time.Sleep(time.Second) - for i := 0; i < 30000; i++ { + for i := 0; i < 30010; i++ { data, err := d.Pop() if err == nil { //fmt.Println(i, string(data.buf), err) @@ -306,7 +312,9 @@ func TestFIFO(t *testing.T) { //fmt.Println("err", err) logs.Warn("err", err) } + //logs.Warn(d.Len()) } + logs.Warn("pop finish") }() go func() { time.Sleep(time.Second * 10) @@ -314,10 +322,10 @@ func TestFIFO(t *testing.T) { go func(i int) { for n := 0; n < 10; n++ { data := new(ListElement) - by := []byte("test " + strconv.Itoa(i) + strconv.Itoa(n)) + by := []byte("test " + strconv.Itoa(i) + " " + strconv.Itoa(n)) // _ = data.New(by, uint16(len(by)), true) //fmt.Println(string((*data).buf), data) - logs.Warn(string((*data).buf), data) + //logs.Warn(string((*data).buf), data) d.Push(data) } }(i) @@ -337,11 +345,12 @@ func TestPriority(t *testing.T) { d.New() go func() { time.Sleep(time.Second) - for i := 0; i < 36000; i++ { + for i := 0; i < 36005; i++ { data := d.Pop() //fmt.Println(i, string(data.buf), err) logs.Warn(i, string(data.Content), data) } + logs.Warn("pop finish") }() go func() { time.Sleep(time.Second * 10) diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 488c616..6ed2dd6 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -1,10 +1,12 @@ package mux import ( + "bytes" "errors" "github.com/cnlh/nps/lib/common" "io" "math" + "runtime" "sync/atomic" "time" "unsafe" @@ -13,7 +15,7 @@ import ( type QueueOp struct { readOp chan struct{} cleanOp chan struct{} - popWait int32 + popWait uint32 } func (Self *QueueOp) New() { @@ -22,7 +24,7 @@ func (Self *QueueOp) New() { } func (Self *QueueOp) allowPop() (closed bool) { - if atomic.CompareAndSwapInt32(&Self.popWait, 1, 0) { + if atomic.CompareAndSwapUint32(&Self.popWait, 1, 0) { select { case Self.readOp <- struct{}{}: return false @@ -44,7 +46,7 @@ type PriorityQueue struct { highestChain *bufChain middleChain *bufChain lowestChain *bufChain - hunger uint8 + starving uint8 } func (Self *PriorityQueue) New() { @@ -73,7 +75,7 @@ func (Self *PriorityQueue) Push(packager *common.MuxPackager) { return } -const maxHunger uint8 = 10 +const maxStarving uint8 = 8 func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { startPop: @@ -82,31 +84,32 @@ startPop: packager = (*common.MuxPackager)(ptr) return } - if Self.hunger < maxHunger { + if Self.starving < maxStarving { ptr, ok = Self.middleChain.popTail() if ok { packager = (*common.MuxPackager)(ptr) - Self.hunger++ + Self.starving++ return } } ptr, ok = Self.lowestChain.popTail() if ok { packager = (*common.MuxPackager)(ptr) - if Self.hunger > 0 { - Self.hunger = uint8(Self.hunger / 2) + if Self.starving > 0 { + Self.starving = uint8(Self.starving / 2) } return } - if Self.hunger > 0 { + if Self.starving > 0 { ptr, ok = Self.middleChain.popTail() if ok { packager = (*common.MuxPackager)(ptr) + Self.starving++ return } } // PriorityQueue is empty, notice Push method - if atomic.CompareAndSwapInt32(&Self.popWait, 0, 1) { + if atomic.CompareAndSwapUint32(&Self.popWait, 0, 1) { select { case <-Self.readOp: goto startPop @@ -133,7 +136,7 @@ func (Self *ListElement) New(buf []byte, l uint16, part bool) (err error) { return nil } -type FIFOQueue struct { +type ReceiveWindowQueue struct { QueueOp chain *bufChain length uint32 @@ -141,21 +144,21 @@ type FIFOQueue struct { timeout time.Time } -func (Self *FIFOQueue) New() { +func (Self *ReceiveWindowQueue) New() { Self.QueueOp.New() Self.chain = new(bufChain) Self.chain.new(64) Self.stopOp = make(chan struct{}, 1) } -func (Self *FIFOQueue) Push(element *ListElement) { +func (Self *ReceiveWindowQueue) Push(element *ListElement) { Self.chain.pushHead(unsafe.Pointer(element)) atomic.AddUint32(&Self.length, uint32(element.l)) Self.allowPop() return } -func (Self *FIFOQueue) Pop() (element *ListElement, err error) { +func (Self *ReceiveWindowQueue) Pop() (element *ListElement, err error) { startPop: ptr, ok := Self.chain.popTail() if ok { @@ -163,7 +166,7 @@ startPop: atomic.AddUint32(&Self.length, ^uint32(element.l-1)) return } - if atomic.CompareAndSwapInt32(&Self.popWait, 0, 1) { + if atomic.CompareAndSwapUint32(&Self.popWait, 0, 1) { t := Self.timeout.Sub(time.Now()) if t <= 0 { t = time.Minute @@ -186,18 +189,62 @@ startPop: goto startPop } -func (Self *FIFOQueue) Len() (n uint32) { +func (Self *ReceiveWindowQueue) Len() (n uint32) { return atomic.LoadUint32(&Self.length) } -func (Self *FIFOQueue) Stop() { +func (Self *ReceiveWindowQueue) Stop() { Self.stopOp <- struct{}{} } -func (Self *FIFOQueue) SetTimeOut(t time.Time) { +func (Self *ReceiveWindowQueue) SetTimeOut(t time.Time) { Self.timeout = t } +type BytesQueue struct { + QueueOp + chain *bufChain + stopOp chan struct{} +} + +func (Self *BytesQueue) New() { + Self.QueueOp.New() + Self.chain = new(bufChain) + Self.chain.new(8) + Self.stopOp = make(chan struct{}, 1) +} + +func (Self *BytesQueue) Push(buf *bytes.Buffer) { + Self.chain.pushHead(unsafe.Pointer(buf)) + Self.allowPop() + return +} + +func (Self *BytesQueue) Pop() (buf *bytes.Buffer, err error) { +startPop: + ptr, ok := Self.chain.popTail() + if ok { + buf = (*bytes.Buffer)(ptr) + return + } + if atomic.CompareAndSwapUint32(&Self.popWait, 0, 1) { + select { + case <-Self.readOp: + goto startPop + case <-Self.cleanOp: + return + case <-Self.stopOp: + err = io.EOF + return + } + } + goto startPop +} + +func (Self *BytesQueue) Stop() { + Self.stopOp <- struct{}{} +} + // https://golang.org/src/sync/poolqueue.go type bufDequeue struct { @@ -224,7 +271,8 @@ type bufDequeue struct { // index has moved beyond it and typ has been set to nil. This // is set to nil atomically by the consumer and read // atomically by the producer. - vals []unsafe.Pointer + vals []unsafe.Pointer + starving uint32 } const dequeueBits = 32 @@ -253,6 +301,10 @@ func (d *bufDequeue) pack(head, tail uint32) uint64 { // queue is full. func (d *bufDequeue) pushHead(val unsafe.Pointer) bool { var slot *unsafe.Pointer + var starve uint8 + if atomic.LoadUint32(&d.starving) > 0 { + runtime.Gosched() + } for { ptrs := atomic.LoadUint64(&d.headTail) head, tail := d.unpack(ptrs) @@ -263,8 +315,15 @@ func (d *bufDequeue) pushHead(val unsafe.Pointer) bool { ptrs2 := d.pack(head+1, tail) if atomic.CompareAndSwapUint64(&d.headTail, ptrs, ptrs2) { slot = &d.vals[head&uint32(len(d.vals)-1)] + if starve >= 3 && atomic.LoadUint32(&d.starving) > 0 { + atomic.StoreUint32(&d.starving, 0) + } break } + starve++ + if starve >= 3 { + atomic.StoreUint32(&d.starving, 1) + } } // The head slot is free, so we own it. *slot = val @@ -321,8 +380,8 @@ type bufChain struct { // tail is the bufDequeue to popTail from. This is accessed // by consumers, so reads and writes must be atomic. - tail *bufChainElt - chainStatus int32 + tail *bufChainElt + newChain uint32 } type bufChainElt struct { @@ -359,30 +418,39 @@ func (c *bufChain) new(initSize int) { } func (c *bufChain) pushHead(val unsafe.Pointer) { +startPush: for { - d := loadPoolChainElt(&c.head) - - if d.pushHead(val) { - return - } - - // The current dequeue is full. Allocate a new one of twice - // the size. - if atomic.CompareAndSwapInt32(&c.chainStatus, 0, 1) { - newSize := len(d.vals) * 2 - if newSize >= dequeueLimit { - // Can't make it any bigger. - newSize = dequeueLimit - } - - d2 := &bufChainElt{prev: d} - d2.vals = make([]unsafe.Pointer, newSize) - d2.pushHead(val) - storePoolChainElt(&d.next, d2) - storePoolChainElt(&c.head, d2) - atomic.StoreInt32(&c.chainStatus, 0) + if atomic.LoadUint32(&c.newChain) > 0 { + runtime.Gosched() + } else { + break } } + + d := loadPoolChainElt(&c.head) + + if d.pushHead(val) { + return + } + + // The current dequeue is full. Allocate a new one of twice + // the size. + if atomic.CompareAndSwapUint32(&c.newChain, 0, 1) { + newSize := len(d.vals) * 2 + if newSize >= dequeueLimit { + // Can't make it any bigger. + newSize = dequeueLimit + } + + d2 := &bufChainElt{prev: d} + d2.vals = make([]unsafe.Pointer, newSize) + d2.pushHead(val) + storePoolChainElt(&c.head, d2) + storePoolChainElt(&d.next, d2) + atomic.StoreUint32(&c.newChain, 0) + return + } + goto startPush } func (c *bufChain) popTail() (unsafe.Pointer, bool) { From 5f58c34c8bd53c584f832ace882fe7d175508994 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sat, 2 Nov 2019 22:59:52 +0800 Subject: [PATCH 28/38] perf test --- go.mod | 1 + go.sum | 2 + lib/common/pool.go | 53 ++++++++++++++ lib/mux/conn.go | 4 +- lib/mux/mux.go | 11 ++- lib/mux/mux_test.go | 122 ++++++++++++++++--------------- lib/mux/queue.go | 172 ++++++++++++++++++++------------------------ 7 files changed, 207 insertions(+), 158 deletions(-) diff --git a/go.mod b/go.mod index 8a19eaf..a540dae 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/klauspost/cpuid v1.2.1 // indirect github.com/klauspost/reedsolomon v1.9.2 // indirect github.com/onsi/gomega v1.5.0 // indirect + github.com/panjf2000/ants/v2 v2.2.2 github.com/pkg/errors v0.8.0 github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 // indirect github.com/shirou/gopsutil v2.18.12+incompatible diff --git a/go.sum b/go.sum index f3a17f4..f0c4d7f 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,8 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/panjf2000/ants/v2 v2.2.2 h1:TWzusBjq/IflXhy+/S6u5wmMLCBdJnB9tPIx9Zmhvok= +github.com/panjf2000/ants/v2 v2.2.2/go.mod h1:1GFm8bV8nyCQvU5K4WvBCTG1/YBFOD2VzjffD8fV55A= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/lib/common/pool.go b/lib/common/pool.go index 240f7f9..5010012 100644 --- a/lib/common/pool.go +++ b/lib/common/pool.go @@ -2,6 +2,8 @@ package common import ( "bytes" + "github.com/panjf2000/ants/v2" + "net" "sync" ) @@ -149,11 +151,62 @@ func (Self *muxPackagerPool) Put(pack *MuxPackager) { Self.pool.Put(pack) } +type connGroup struct { + src net.Conn + dst net.Conn + wg *sync.WaitGroup +} + +func newConnGroup(src net.Conn, dst net.Conn, wg *sync.WaitGroup) connGroup { + return connGroup{ + src: src, + dst: dst, + wg: wg, + } +} + +func copyConnGroup(group interface{}) { + cg, ok := group.(connGroup) + if !ok { + return + } + _, err := CopyBuffer(cg.src, cg.dst) + if err != nil { + cg.src.Close() + cg.dst.Close() + //logs.Warn("close npc by copy from nps", err, c.connId) + } + cg.wg.Done() +} + +type Conns struct { + conn1 net.Conn + conn2 net.Conn +} + +func NewConns(c1 net.Conn, c2 net.Conn) Conns { + return Conns{ + conn1: c1, + conn2: c2, + } +} + +func copyConns(group interface{}) { + conns := group.(Conns) + wg := new(sync.WaitGroup) + wg.Add(2) + _ = connCopyPool.Invoke(newConnGroup(conns.conn1, conns.conn2, wg)) + _ = connCopyPool.Invoke(newConnGroup(conns.conn2, conns.conn1, wg)) + wg.Wait() +} + var once = sync.Once{} var BuffPool = bufferPool{} var CopyBuff = copyBufferPool{} var MuxPack = muxPackagerPool{} var WindowBuff = windowBufferPool{} +var connCopyPool, _ = ants.NewPoolWithFunc(200000, copyConnGroup, ants.WithNonblocking(false)) +var CopyConnsPool, _ = ants.NewPoolWithFunc(100000, copyConns, ants.WithNonblocking(false)) func newPool() { BuffPool.New() diff --git a/lib/mux/conn.go b/lib/mux/conn.go index f3217d8..0ba6f90 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -156,7 +156,7 @@ type ReceiveWindow struct { bufQueue ReceiveWindowQueue element *ListElement readLength uint32 - readOp chan struct{} + //readOp chan struct{} readWait bool windowFull uint32 count int8 @@ -167,7 +167,7 @@ type ReceiveWindow struct { func (Self *ReceiveWindow) New(mux *Mux) { // initial a window for receive - Self.readOp = make(chan struct{}) + //Self.readOp = make(chan struct{}) Self.bufQueue.New() //Self.bw = new(bandwidth) Self.element = new(ListElement) diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 8a6086c..6a2d6b6 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -5,7 +5,6 @@ import ( "io" "math" "net" - "sync" "sync/atomic" "time" @@ -29,7 +28,7 @@ type Mux struct { connType string writeQueue PriorityQueue //bufQueue BytesQueue - sync.Mutex + //sync.Mutex } func NewMux(c net.Conn, connType string) *Mux { @@ -216,7 +215,7 @@ func (s *Mux) pingReturn() { if latency < 0.5 && latency > 0 { s.latency = latency } - //logs.Warn("latency", s.latency) + logs.Warn("latency", s.latency) common.WindowBuff.Put(data) } }() @@ -242,15 +241,19 @@ func (s *Mux) readSession() { case common.MUX_NEW_CONN: //new connection connection := NewConn(pack.Id, s, "npc ") s.connMap.Set(pack.Id, connection) //it has been set before send ok + //go func(connection *conn) { s.newConnCh <- connection s.sendInfo(common.MUX_NEW_CONN_OK, connection.connId, nil) + //}(connection) continue case common.MUX_PING_FLAG: //ping s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, pack.Content) common.WindowBuff.Put(pack.Content) continue case common.MUX_PING_RETURN: + //go func(content []byte) { s.pingCh <- pack.Content + //}(pack.Content) continue } if connection, ok := s.connMap.Get(pack.Id); ok && !connection.isClose { @@ -275,8 +278,10 @@ func (s *Mux) readSession() { continue case common.MUX_CONN_CLOSE: //close the connection s.connMap.Delete(pack.Id) + //go func(connection *conn) { connection.closeFlag = true connection.receiveWindow.Stop() // close signal to receive window + //}(connection) continue } } else if pack.Flag == common.MUX_CONN_CLOSE { diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index e3f9dcc..d3061ab 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -11,7 +11,6 @@ import ( "net/http/httputil" _ "net/http/pprof" "strconv" - "sync" "testing" "time" "unsafe" @@ -30,6 +29,7 @@ func TestNewMux(t *testing.T) { logs.SetLogFuncCallDepth(3) server() client() + //poolConnCopy, _ := ants.NewPoolWithFunc(200000, common.copyConn, ants.WithNonblocking(false)) time.Sleep(time.Second * 3) go func() { m2 := NewMux(conn2, "tcp") @@ -49,31 +49,34 @@ func TestNewMux(t *testing.T) { } //c2.(*net.TCPConn).SetReadBuffer(0) //c2.(*net.TCPConn).SetReadBuffer(0) - go func(c2 net.Conn, c *conn) { - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - _, err = common.CopyBuffer(c2, c) - if err != nil { - c2.Close() - c.Close() - //logs.Warn("close npc by copy from nps", err, c.connId) - } - wg.Done() - }() - wg.Add(1) - go func() { - _, err = common.CopyBuffer(c, c2) - if err != nil { - c2.Close() - c.Close() - //logs.Warn("close npc by copy from server", err, c.connId) - } - wg.Done() - }() - //logs.Warn("npc wait") - wg.Wait() - }(c2, c.(*conn)) + _ = common.CopyConnsPool.Invoke(common.NewConns(c2, c)) + //go func(c2 net.Conn, c *conn) { + // wg := new(sync.WaitGroup) + // wg.Add(2) + // _ = poolConnCopy.Invoke(common.newConnGroup(c2, c, wg)) + // //go func() { + // // _, err = common.CopyBuffer(c2, c) + // // if err != nil { + // // c2.Close() + // // c.Close() + // // //logs.Warn("close npc by copy from nps", err, c.connId) + // // } + // // wg.Done() + // //}() + // //wg.Add(1) + // _ = poolConnCopy.Invoke(common.newConnGroup(c, c2, wg)) + // //go func() { + // // _, err = common.CopyBuffer(c, c2) + // // if err != nil { + // // c2.Close() + // // c.Close() + // // //logs.Warn("close npc by copy from server", err, c.connId) + // // } + // // wg.Done() + // //}() + // //logs.Warn("npc wait") + // wg.Wait() + //}(c2, c.(*conn)) } }() @@ -99,23 +102,30 @@ func TestNewMux(t *testing.T) { continue } //logs.Warn("nps new conn success ", tmpCpnn.connId) - go func(tmpCpnn *conn, conns net.Conn) { - go func() { - _, err := common.CopyBuffer(tmpCpnn, conns) - if err != nil { - conns.Close() - tmpCpnn.Close() - //logs.Warn("close nps by copy from user", tmpCpnn.connId, err) - } - }() - //time.Sleep(time.Second) - _, err = common.CopyBuffer(conns, tmpCpnn) - if err != nil { - conns.Close() - tmpCpnn.Close() - //logs.Warn("close nps by copy from npc ", tmpCpnn.connId, err) - } - }(tmpCpnn, conns) + _ = common.CopyConnsPool.Invoke(common.NewConns(tmpCpnn, conns)) + //go func(tmpCpnn *conn, conns net.Conn) { + // wg := new(sync.WaitGroup) + // wg.Add(2) + // _ = poolConnCopy.Invoke(common.newConnGroup(tmpCpnn, conns, wg)) + // //go func() { + // // _, err := common.CopyBuffer(tmpCpnn, conns) + // // if err != nil { + // // conns.Close() + // // tmpCpnn.Close() + // // //logs.Warn("close nps by copy from user", tmpCpnn.connId, err) + // // } + // //}() + // //wg.Add(1) + // _ = poolConnCopy.Invoke(common.newConnGroup(conns, tmpCpnn, wg)) + // //time.Sleep(time.Second) + // //_, err = common.CopyBuffer(conns, tmpCpnn) + // //if err != nil { + // // conns.Close() + // // tmpCpnn.Close() + // // //logs.Warn("close nps by copy from npc ", tmpCpnn.connId, err) + // //} + // wg.Wait() + //}(tmpCpnn, conns) } }() @@ -180,7 +190,7 @@ Connection: keep-alive } func test_raw(k int) { - for i := 0; i < 10; i++ { + for i := 0; i < 1; i++ { ti := time.Now() conn, err := net.Dial("tcp", "127.0.0.1:7777") if err != nil { @@ -303,7 +313,7 @@ func TestFIFO(t *testing.T) { d.New() go func() { time.Sleep(time.Second) - for i := 0; i < 30010; i++ { + for i := 0; i < 300100; i++ { data, err := d.Pop() if err == nil { //fmt.Println(i, string(data.buf), err) @@ -318,17 +328,13 @@ func TestFIFO(t *testing.T) { }() go func() { time.Sleep(time.Second * 10) - for i := 0; i < 3000; i++ { - go func(i int) { - for n := 0; n < 10; n++ { - data := new(ListElement) - by := []byte("test " + strconv.Itoa(i) + " " + strconv.Itoa(n)) // - _ = data.New(by, uint16(len(by)), true) - //fmt.Println(string((*data).buf), data) - //logs.Warn(string((*data).buf), data) - d.Push(data) - } - }(i) + for i := 0; i < 300000; i++ { + data := new(ListElement) + by := []byte("test " + strconv.Itoa(i) + " ") // + _ = data.New(by, uint16(len(by)), true) + //fmt.Println(string((*data).buf), data) + //logs.Warn(string((*data).buf), data) + d.Push(data) } }() time.Sleep(time.Second * 100000) @@ -345,7 +351,7 @@ func TestPriority(t *testing.T) { d.New() go func() { time.Sleep(time.Second) - for i := 0; i < 36005; i++ { + for i := 0; i < 360050; i++ { data := d.Pop() //fmt.Println(i, string(data.buf), err) logs.Warn(i, string(data.Content), data) @@ -354,7 +360,7 @@ func TestPriority(t *testing.T) { }() go func() { time.Sleep(time.Second * 10) - for i := 0; i < 3000; i++ { + for i := 0; i < 30000; i++ { go func(i int) { for n := 0; n < 10; n++ { data := new(common.MuxPackager) diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 6ed2dd6..e3c39a1 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -1,52 +1,24 @@ package mux import ( - "bytes" "errors" "github.com/cnlh/nps/lib/common" "io" "math" "runtime" + "sync" "sync/atomic" "time" "unsafe" ) -type QueueOp struct { - readOp chan struct{} - cleanOp chan struct{} - popWait uint32 -} - -func (Self *QueueOp) New() { - Self.readOp = make(chan struct{}) - Self.cleanOp = make(chan struct{}, 2) -} - -func (Self *QueueOp) allowPop() (closed bool) { - if atomic.CompareAndSwapUint32(&Self.popWait, 1, 0) { - select { - case Self.readOp <- struct{}{}: - return false - case <-Self.cleanOp: - return true - } - } - return -} - -func (Self *QueueOp) Clean() { - Self.cleanOp <- struct{}{} - Self.cleanOp <- struct{}{} - close(Self.cleanOp) -} - type PriorityQueue struct { - QueueOp highestChain *bufChain middleChain *bufChain lowestChain *bufChain starving uint8 + stop bool + cond *sync.Cond } func (Self *PriorityQueue) New() { @@ -56,7 +28,8 @@ func (Self *PriorityQueue) New() { Self.middleChain.new(32) Self.lowestChain = new(bufChain) Self.lowestChain.new(256) - Self.QueueOp.New() + locker := new(sync.Mutex) + Self.cond = sync.NewCond(locker) } func (Self *PriorityQueue) Push(packager *common.MuxPackager) { @@ -71,14 +44,44 @@ func (Self *PriorityQueue) Push(packager *common.MuxPackager) { default: Self.lowestChain.pushHead(unsafe.Pointer(packager)) } - Self.allowPop() + //atomic.AddUint32(&Self.count, 1) + Self.cond.Signal() return } const maxStarving uint8 = 8 func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { -startPop: + // PriorityQueue is empty, notice Push method + var iter bool + for { + packager = Self.pop() + if packager != nil { + return + } + if Self.stop { + return + } + if iter { + break + } + iter = true + runtime.Gosched() + } + Self.cond.L.Lock() + defer Self.cond.L.Unlock() + for packager = Self.pop(); packager == nil; { + if Self.stop { + return + } + Self.cond.Wait() + packager = Self.pop() + } + //atomic.AddUint32(&Self.count, ^uint32(0)) + return +} + +func (Self *PriorityQueue) pop() (packager *common.MuxPackager) { ptr, ok := Self.highestChain.popTail() if ok { packager = (*common.MuxPackager)(ptr) @@ -108,16 +111,12 @@ startPop: return } } - // PriorityQueue is empty, notice Push method - if atomic.CompareAndSwapUint32(&Self.popWait, 0, 1) { - select { - case <-Self.readOp: - goto startPop - case <-Self.cleanOp: - return nil - } - } - goto startPop + return +} + +func (Self *PriorityQueue) Stop() { + Self.stop = true + Self.cond.Broadcast() } type ListElement struct { @@ -137,18 +136,19 @@ func (Self *ListElement) New(buf []byte, l uint16, part bool) (err error) { } type ReceiveWindowQueue struct { - QueueOp chain *bufChain length uint32 stopOp chan struct{} + readOp chan struct{} + popWait uint32 timeout time.Time } func (Self *ReceiveWindowQueue) New() { - Self.QueueOp.New() + Self.readOp = make(chan struct{}) Self.chain = new(bufChain) Self.chain.new(64) - Self.stopOp = make(chan struct{}, 1) + Self.stopOp = make(chan struct{}, 2) } func (Self *ReceiveWindowQueue) Push(element *ListElement) { @@ -158,15 +158,30 @@ func (Self *ReceiveWindowQueue) Push(element *ListElement) { return } -func (Self *ReceiveWindowQueue) Pop() (element *ListElement, err error) { -startPop: +func (Self *ReceiveWindowQueue) pop() (element *ListElement) { ptr, ok := Self.chain.popTail() if ok { element = (*ListElement)(ptr) atomic.AddUint32(&Self.length, ^uint32(element.l-1)) return } + return +} + +func (Self *ReceiveWindowQueue) Pop() (element *ListElement, err error) { + var iter bool +startPop: + element = Self.pop() + if element != nil { + return + } + if !iter { + iter = true + runtime.Gosched() + goto startPop + } if atomic.CompareAndSwapUint32(&Self.popWait, 0, 1) { + iter = false t := Self.timeout.Sub(time.Now()) if t <= 0 { t = time.Minute @@ -176,8 +191,6 @@ startPop: select { case <-Self.readOp: goto startPop - case <-Self.cleanOp: - return case <-Self.stopOp: err = io.EOF return @@ -189,62 +202,31 @@ startPop: goto startPop } +func (Self *ReceiveWindowQueue) allowPop() (closed bool) { + if atomic.CompareAndSwapUint32(&Self.popWait, 1, 0) { + select { + case Self.readOp <- struct{}{}: + return false + case <-Self.stopOp: + return true + } + } + return +} + func (Self *ReceiveWindowQueue) Len() (n uint32) { return atomic.LoadUint32(&Self.length) } func (Self *ReceiveWindowQueue) Stop() { Self.stopOp <- struct{}{} + Self.stopOp <- struct{}{} } func (Self *ReceiveWindowQueue) SetTimeOut(t time.Time) { Self.timeout = t } -type BytesQueue struct { - QueueOp - chain *bufChain - stopOp chan struct{} -} - -func (Self *BytesQueue) New() { - Self.QueueOp.New() - Self.chain = new(bufChain) - Self.chain.new(8) - Self.stopOp = make(chan struct{}, 1) -} - -func (Self *BytesQueue) Push(buf *bytes.Buffer) { - Self.chain.pushHead(unsafe.Pointer(buf)) - Self.allowPop() - return -} - -func (Self *BytesQueue) Pop() (buf *bytes.Buffer, err error) { -startPop: - ptr, ok := Self.chain.popTail() - if ok { - buf = (*bytes.Buffer)(ptr) - return - } - if atomic.CompareAndSwapUint32(&Self.popWait, 0, 1) { - select { - case <-Self.readOp: - goto startPop - case <-Self.cleanOp: - return - case <-Self.stopOp: - err = io.EOF - return - } - } - goto startPop -} - -func (Self *BytesQueue) Stop() { - Self.stopOp <- struct{}{} -} - // https://golang.org/src/sync/poolqueue.go type bufDequeue struct { From f362c96e1ee801e41703de2fea93c3239593122e Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sat, 9 Nov 2019 23:02:29 +0800 Subject: [PATCH 29/38] fine mux, add goroutine pool --- lib/common/pool.go | 67 ++++-------- lib/conn/conn.go | 37 ++++--- lib/goroutine/pool.go | 73 +++++++++++++ lib/mux/conn.go | 247 +++++++++++++++++++++++++----------------- lib/mux/mux.go | 2 +- lib/mux/mux_test.go | 31 +++--- lib/mux/queue.go | 123 +++++++++++---------- 7 files changed, 352 insertions(+), 228 deletions(-) create mode 100644 lib/goroutine/pool.go diff --git a/lib/common/pool.go b/lib/common/pool.go index 5010012..5d4da00 100644 --- a/lib/common/pool.go +++ b/lib/common/pool.go @@ -2,8 +2,6 @@ package common import ( "bytes" - "github.com/panjf2000/ants/v2" - "net" "sync" ) @@ -151,53 +149,34 @@ func (Self *muxPackagerPool) Put(pack *MuxPackager) { Self.pool.Put(pack) } -type connGroup struct { - src net.Conn - dst net.Conn - wg *sync.WaitGroup +type ListElement struct { + Buf []byte + L uint16 + Part bool } -func newConnGroup(src net.Conn, dst net.Conn, wg *sync.WaitGroup) connGroup { - return connGroup{ - src: src, - dst: dst, - wg: wg, +type listElementPool struct { + pool sync.Pool +} + +func (Self *listElementPool) New() { + Self.pool = sync.Pool{ + New: func() interface{} { + element := ListElement{} + return &element + }, } } -func copyConnGroup(group interface{}) { - cg, ok := group.(connGroup) - if !ok { - return - } - _, err := CopyBuffer(cg.src, cg.dst) - if err != nil { - cg.src.Close() - cg.dst.Close() - //logs.Warn("close npc by copy from nps", err, c.connId) - } - cg.wg.Done() +func (Self *listElementPool) Get() *ListElement { + return Self.pool.Get().(*ListElement) } -type Conns struct { - conn1 net.Conn - conn2 net.Conn -} - -func NewConns(c1 net.Conn, c2 net.Conn) Conns { - return Conns{ - conn1: c1, - conn2: c2, - } -} - -func copyConns(group interface{}) { - conns := group.(Conns) - wg := new(sync.WaitGroup) - wg.Add(2) - _ = connCopyPool.Invoke(newConnGroup(conns.conn1, conns.conn2, wg)) - _ = connCopyPool.Invoke(newConnGroup(conns.conn2, conns.conn1, wg)) - wg.Wait() +func (Self *listElementPool) Put(element *ListElement) { + element.L = 0 + element.Buf = nil + element.Part = false + Self.pool.Put(element) } var once = sync.Once{} @@ -205,14 +184,14 @@ var BuffPool = bufferPool{} var CopyBuff = copyBufferPool{} var MuxPack = muxPackagerPool{} var WindowBuff = windowBufferPool{} -var connCopyPool, _ = ants.NewPoolWithFunc(200000, copyConnGroup, ants.WithNonblocking(false)) -var CopyConnsPool, _ = ants.NewPoolWithFunc(100000, copyConns, ants.WithNonblocking(false)) +var ListElementPool = listElementPool{} func newPool() { BuffPool.New() CopyBuff.New() MuxPack.New() WindowBuff.New() + ListElementPool.New() } func init() { diff --git a/lib/conn/conn.go b/lib/conn/conn.go index 7946c0d..9f0c397 100755 --- a/lib/conn/conn.go +++ b/lib/conn/conn.go @@ -6,13 +6,14 @@ import ( "encoding/binary" "encoding/json" "errors" + "github.com/astaxie/beego/logs" + "github.com/cnlh/nps/lib/goroutine" "io" "net" "net/http" "net/url" "strconv" "strings" - "sync" "time" "github.com/cnlh/nps/lib/common" @@ -350,25 +351,29 @@ func SetUdpSession(sess *kcp.UDPSession) { //conn1 mux conn func CopyWaitGroup(conn1, conn2 net.Conn, crypt bool, snappy bool, rate *rate.Rate, flow *file.Flow, isServer bool, rb []byte) { - var in, out int64 - var wg sync.WaitGroup + //var in, out int64 + //var wg sync.WaitGroup connHandle := GetConn(conn1, crypt, snappy, rate, isServer) if rb != nil { connHandle.Write(rb) } - go func(in *int64) { - wg.Add(1) - *in, _ = common.CopyBuffer(connHandle, conn2) - connHandle.Close() - conn2.Close() - wg.Done() - }(&in) - out, _ = common.CopyBuffer(conn2, connHandle) - connHandle.Close() - conn2.Close() - wg.Wait() - if flow != nil { - flow.Add(in, out) + //go func(in *int64) { + // wg.Add(1) + // *in, _ = common.CopyBuffer(connHandle, conn2) + // connHandle.Close() + // conn2.Close() + // wg.Done() + //}(&in) + //out, _ = common.CopyBuffer(conn2, connHandle) + //connHandle.Close() + //conn2.Close() + //wg.Wait() + //if flow != nil { + // flow.Add(in, out) + //} + err := goroutine.CopyConnsPool.Invoke(goroutine.NewConns(connHandle, conn2, flow)) + if err != nil { + logs.Error(err) } } diff --git a/lib/goroutine/pool.go b/lib/goroutine/pool.go new file mode 100644 index 0000000..287c711 --- /dev/null +++ b/lib/goroutine/pool.go @@ -0,0 +1,73 @@ +package goroutine + +import ( + "github.com/cnlh/nps/lib/common" + "github.com/cnlh/nps/lib/file" + "github.com/panjf2000/ants/v2" + "io" + "net" + "sync" +) + +type connGroup struct { + src io.ReadWriteCloser + dst io.ReadWriteCloser + wg *sync.WaitGroup + n *int64 +} + +func newConnGroup(dst, src io.ReadWriteCloser, wg *sync.WaitGroup, n *int64) connGroup { + return connGroup{ + src: src, + dst: dst, + wg: wg, + n: n, + } +} + +func copyConnGroup(group interface{}) { + cg, ok := group.(connGroup) + if !ok { + return + } + var err error + *cg.n, err = common.CopyBuffer(cg.dst, cg.src) + if err != nil { + cg.src.Close() + cg.dst.Close() + //logs.Warn("close npc by copy from nps", err, c.connId) + } + cg.wg.Done() +} + +type Conns struct { + conn1 io.ReadWriteCloser // mux connection + conn2 net.Conn // outside connection + flow *file.Flow +} + +func NewConns(c1 io.ReadWriteCloser, c2 net.Conn, flow *file.Flow) Conns { + return Conns{ + conn1: c1, + conn2: c2, + flow: flow, + } +} + +func copyConns(group interface{}) { + conns := group.(Conns) + wg := new(sync.WaitGroup) + wg.Add(2) + var in, out int64 + _ = connCopyPool.Invoke(newConnGroup(conns.conn1, conns.conn2, wg, &in)) + // outside to mux : incoming + _ = connCopyPool.Invoke(newConnGroup(conns.conn2, conns.conn1, wg, &out)) + // mux to outside : outgoing + wg.Wait() + if conns.flow != nil { + conns.flow.Add(in, out) + } +} + +var connCopyPool, _ = ants.NewPoolWithFunc(200000, copyConnGroup, ants.WithNonblocking(false)) +var CopyConnsPool, _ = ants.NewPoolWithFunc(100000, copyConns, ants.WithNonblocking(false)) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 0ba6f90..cb982e8 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -4,6 +4,7 @@ import ( "errors" "io" "net" + "runtime" "sync" "sync/atomic" "time" @@ -57,7 +58,12 @@ func (s *conn) Read(buf []byte) (n int, err error) { return 0, nil } // waiting for takeout from receive window finish or timeout + //now := time.Now() n, err = s.receiveWindow.Read(buf, s.connId) + //t := time.Now().Sub(now) + //if t.Seconds() > 0.5 { + //logs.Warn("conn read long", n, t.Seconds()) + //} //var errstr string //if err == nil { // errstr = "err:nil" @@ -82,7 +88,12 @@ func (s *conn) Write(buf []byte) (n int, err error) { return 0, nil } //logs.Warn("write buf", len(buf)) + //now := time.Now() n, err = s.sendWindow.WriteFull(buf, s.connId) + //t := time.Now().Sub(now) + //if t.Seconds() > 0.5 { + // logs.Warn("conn write long", n, t.Seconds()) + //} return } @@ -133,11 +144,25 @@ func (s *conn) SetWriteDeadline(t time.Time) error { } type window struct { - off uint32 - maxSize uint32 - closeOp bool - closeOpCh chan struct{} - mux *Mux + off uint32 + maxSize uint32 + closeOp bool + closeOpCh chan struct{} + remainingWait uint64 + mux *Mux +} + +func (Self *window) unpack(ptrs uint64) (remaining, wait uint32) { + const mask = 1<> dequeueBits) & mask) + wait = uint32(ptrs & mask) + return +} + +func (Self *window) pack(remaining, wait uint32) uint64 { + const mask = 1< 0 { + n = uint32(l) + } + return } func (Self *ReceiveWindow) calcSize() { @@ -194,8 +212,9 @@ func (Self *ReceiveWindow) calcSize() { if n < 8192 { n = 8192 } - if n < Self.bufQueue.Len() { - n = Self.bufQueue.Len() + bufLen := Self.bufQueue.Len() + if n < bufLen { + n = bufLen } // set the minimal size if n > 2*Self.maxSize { @@ -210,28 +229,39 @@ func (Self *ReceiveWindow) calcSize() { Self.count = -10 } Self.count += 1 + return } func (Self *ReceiveWindow) Write(buf []byte, l uint16, part bool, id int32) (err error) { if Self.closeOp { return errors.New("conn.receiveWindow: write on closed window") } - element := new(ListElement) - err = element.New(buf, l, part) + element, err := NewListElement(buf, l, part) //logs.Warn("push the buf", len(buf), l, (&element).l) if err != nil { return } - Self.bufQueue.Push(element) // must push data before allow read - //logs.Warn("read session calc size ", Self.maxSize) - // calculating the receive window size - Self.calcSize() - //logs.Warn("read session calc size finish", Self.maxSize) - if Self.remainingSize() == 0 { - atomic.StoreUint32(&Self.windowFull, 1) - //logs.Warn("window full true", Self.windowFull) + Self.calcSize() // calculate the max window size + var wait uint32 +start: + ptrs := atomic.LoadUint64(&Self.remainingWait) + _, wait = Self.unpack(ptrs) + newRemaining := Self.remainingSize(l) + // calculate the remaining window size now, plus the element we will push + if newRemaining == 0 { + //logs.Warn("window full true", remaining) + wait = 1 + } + if !atomic.CompareAndSwapUint64(&Self.remainingWait, ptrs, Self.pack(0, wait)) { + goto start + // another goroutine change the status, make sure shall we need wait + } + Self.bufQueue.Push(element) + // status check finish, now we can push the element into the queue + if wait == 0 { + Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, newRemaining) + // send the remaining window size, not including zero size } - Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, Self.maxSize, Self.readSize()) return nil } @@ -243,10 +273,10 @@ func (Self *ReceiveWindow) Read(p []byte, id int32) (n int, err error) { l := 0 //logs.Warn("receive window read off, element.l", Self.off, Self.element.l) copyData: - //Self.bw.StartRead() - if Self.off == uint32(Self.element.l) { + if Self.off == uint32(Self.element.L) { // on the first Read method invoked, Self.off and Self.element.l // both zero value + common.ListElementPool.Put(Self.element) Self.element, err = Self.bufQueue.Pop() // if the queue is empty, Pop method will wait until one element push // into the queue successful, or timeout. @@ -258,38 +288,44 @@ copyData: } //logs.Warn("pop element", Self.element.l, Self.element.part) } - l = copy(p[pOff:], Self.element.buf[Self.off:Self.element.l]) - //Self.bw.SetCopySize(l) + l = copy(p[pOff:], Self.element.Buf[Self.off:Self.element.L]) pOff += l Self.off += uint32(l) - atomic.AddUint32(&Self.readLength, uint32(l)) //logs.Warn("window read length buf len", Self.readLength, Self.bufQueue.Len()) n += l l = 0 - //Self.bw.EndRead() - if Self.off == uint32(Self.element.l) { + if Self.off == uint32(Self.element.L) { //logs.Warn("put the element end ", string(Self.element.buf[:15])) - common.WindowBuff.Put(Self.element.buf) - Self.sendStatus(id) + common.WindowBuff.Put(Self.element.Buf) + Self.sendStatus(id, Self.element.L) + // check the window full status } - if pOff < len(p) && Self.element.part { + if pOff < len(p) && Self.element.Part { // element is a part of the segments, trying to fill up buf p goto copyData } return // buf p is full or all of segments in buf, return } -func (Self *ReceiveWindow) sendStatus(id int32) { - if Self.bufQueue.Len() == 0 { - // window is full before read or empty now - Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, atomic.LoadUint32(&Self.maxSize), Self.readSize()) - // acknowledge other side, have empty some receive window space - //} +func (Self *ReceiveWindow) sendStatus(id int32, l uint16) { + var remaining, wait uint32 + for { + ptrs := atomic.LoadUint64(&Self.remainingWait) + remaining, wait = Self.unpack(ptrs) + remaining += uint32(l) + if atomic.CompareAndSwapUint64(&Self.remainingWait, ptrs, Self.pack(remaining, 0)) { + break + } + runtime.Gosched() + // another goroutine change remaining or wait status, make sure + // we need acknowledge other side } - if atomic.LoadUint32(&Self.windowFull) > 0 && Self.remainingSize() > 0 { - atomic.StoreUint32(&Self.windowFull, 0) - Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, atomic.LoadUint32(&Self.maxSize), Self.readSize()) + // now we get the current window status success + if wait == 1 { + //logs.Warn("send the wait status", remaining) + Self.mux.sendInfo(common.MUX_MSG_SEND_OK, id, atomic.LoadUint32(&Self.maxSize), remaining) } + return } func (Self *ReceiveWindow) SetTimeOut(t time.Time) { @@ -308,17 +344,16 @@ func (Self *ReceiveWindow) CloseWindow() { } type SendWindow struct { - buf []byte - sentLength uint32 - setSizeCh chan struct{} - setSizeWait uint32 - timeout time.Time + buf []byte + setSizeCh chan struct{} + timeout time.Time window } func (Self *SendWindow) New(mux *Mux) { Self.setSizeCh = make(chan struct{}) Self.maxSize = 4096 + atomic.AddUint64(&Self.remainingWait, uint64(4096)< common.MAXIMUM_SEGMENT_SIZE { sendSize = common.MAXIMUM_SEGMENT_SIZE //logs.Warn("cut buf by mss") } else { sendSize = uint32(len(Self.buf[Self.off:])) } - if Self.RemainingSize() < sendSize { + if remaining < sendSize { // usable window size is small than // window MAXIMUM_SEGMENT_SIZE or send buf left - sendSize = Self.RemainingSize() + sendSize = remaining //logs.Warn("cut buf by remainingsize", sendSize, len(Self.buf[Self.off:])) } //logs.Warn("send size", sendSize) @@ -412,7 +464,7 @@ func (Self *SendWindow) WriteTo() (p []byte, sendSize uint32, part bool, err err } p = Self.buf[Self.off : sendSize+Self.off] Self.off += sendSize - atomic.AddUint32(&Self.sentLength, sendSize) + Self.sent(sendSize) return } @@ -439,6 +491,7 @@ func (Self *SendWindow) waitReceiveWindow() (err error) { func (Self *SendWindow) WriteFull(buf []byte, id int32) (n int, err error) { Self.SetSendBuf(buf) // set the buf to send window + //logs.Warn("set the buf to send window") var bufSeg []byte var part bool var l uint32 diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 6a2d6b6..585c980 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -215,7 +215,7 @@ func (s *Mux) pingReturn() { if latency < 0.5 && latency > 0 { s.latency = latency } - logs.Warn("latency", s.latency) + //logs.Warn("latency", s.latency) common.WindowBuff.Put(data) } }() diff --git a/lib/mux/mux_test.go b/lib/mux/mux_test.go index d3061ab..151def1 100644 --- a/lib/mux/mux_test.go +++ b/lib/mux/mux_test.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "github.com/cnlh/nps/lib/common" + "github.com/cnlh/nps/lib/goroutine" "io" "log" "net" @@ -49,7 +50,7 @@ func TestNewMux(t *testing.T) { } //c2.(*net.TCPConn).SetReadBuffer(0) //c2.(*net.TCPConn).SetReadBuffer(0) - _ = common.CopyConnsPool.Invoke(common.NewConns(c2, c)) + _ = goroutine.CopyConnsPool.Invoke(goroutine.NewConns(c, c2, nil)) //go func(c2 net.Conn, c *conn) { // wg := new(sync.WaitGroup) // wg.Add(2) @@ -102,7 +103,7 @@ func TestNewMux(t *testing.T) { continue } //logs.Warn("nps new conn success ", tmpCpnn.connId) - _ = common.CopyConnsPool.Invoke(common.NewConns(tmpCpnn, conns)) + _ = goroutine.CopyConnsPool.Invoke(goroutine.NewConns(tmpCpnn, conns, nil)) //go func(tmpCpnn *conn, conns net.Conn) { // wg := new(sync.WaitGroup) // wg.Add(2) @@ -131,9 +132,9 @@ func TestNewMux(t *testing.T) { //go NewLogServer() time.Sleep(time.Second * 5) - for i := 0; i < 1000; i++ { - go test_raw(i) - } + //for i := 0; i < 1; i++ { + // go test_raw(i) + //} //test_request() for { @@ -166,7 +167,7 @@ func client() { func test_request() { conn, _ := net.Dial("tcp", "127.0.0.1:7777") - for { + for i := 0; i < 1000; i++ { conn.Write([]byte(`GET / HTTP/1.1 Host: 127.0.0.1:7777 Connection: keep-alive @@ -185,19 +186,20 @@ Connection: keep-alive break } fmt.Println(string(b[:20]), err) - time.Sleep(time.Second) + //time.Sleep(time.Second) } + logs.Warn("finish") } func test_raw(k int) { - for i := 0; i < 1; i++ { + for i := 0; i < 1000; i++ { ti := time.Now() conn, err := net.Dial("tcp", "127.0.0.1:7777") if err != nil { logs.Warn("conn dial err", err) } tid := time.Now() - conn.Write([]byte(`GET / HTTP/1.1 + conn.Write([]byte(`GET /videojs5/video.js HTTP/1.1 Host: 127.0.0.1:7777 @@ -227,6 +229,7 @@ Host: 127.0.0.1:7777 logs.Warn("n loss", n, string(buf)) } } + logs.Warn("finish") } func TestNewConn(t *testing.T) { @@ -313,11 +316,12 @@ func TestFIFO(t *testing.T) { d.New() go func() { time.Sleep(time.Second) - for i := 0; i < 300100; i++ { + for i := 0; i < 1001; i++ { data, err := d.Pop() if err == nil { //fmt.Println(i, string(data.buf), err) - logs.Warn(i, string(data.buf), err) + logs.Warn(i, string(data.Buf), err) + common.ListElementPool.Put(data) } else { //fmt.Println("err", err) logs.Warn("err", err) @@ -328,10 +332,9 @@ func TestFIFO(t *testing.T) { }() go func() { time.Sleep(time.Second * 10) - for i := 0; i < 300000; i++ { - data := new(ListElement) + for i := 0; i < 1000; i++ { by := []byte("test " + strconv.Itoa(i) + " ") // - _ = data.New(by, uint16(len(by)), true) + data, _ := NewListElement(by, uint16(len(by)), true) //fmt.Println(string((*data).buf), data) //logs.Warn(string((*data).buf), data) d.Push(data) diff --git a/lib/mux/queue.go b/lib/mux/queue.go index e3c39a1..4790779 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -44,7 +44,6 @@ func (Self *PriorityQueue) Push(packager *common.MuxPackager) { default: Self.lowestChain.pushHead(unsafe.Pointer(packager)) } - //atomic.AddUint32(&Self.count, 1) Self.cond.Signal() return } @@ -52,7 +51,6 @@ func (Self *PriorityQueue) Push(packager *common.MuxPackager) { const maxStarving uint8 = 8 func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { - // PriorityQueue is empty, notice Push method var iter bool for { packager = Self.pop() @@ -64,6 +62,7 @@ func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { } if iter { break + // trying to pop twice } iter = true runtime.Gosched() @@ -74,10 +73,12 @@ func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { if Self.stop { return } + //logs.Warn("queue into wait") Self.cond.Wait() + // wait for it with no more iter packager = Self.pop() + //logs.Warn("queue wait finish", packager) } - //atomic.AddUint32(&Self.count, ^uint32(0)) return } @@ -88,6 +89,7 @@ func (Self *PriorityQueue) pop() (packager *common.MuxPackager) { return } if Self.starving < maxStarving { + // not pop too much, lowestChain will wait too long ptr, ok = Self.middleChain.popTail() if ok { packager = (*common.MuxPackager)(ptr) @@ -119,29 +121,27 @@ func (Self *PriorityQueue) Stop() { Self.cond.Broadcast() } -type ListElement struct { - buf []byte - l uint16 - part bool -} - -func (Self *ListElement) New(buf []byte, l uint16, part bool) (err error) { +func NewListElement(buf []byte, l uint16, part bool) (element *common.ListElement, err error) { if uint16(len(buf)) != l { - return errors.New("ListElement: buf length not match") + err = errors.New("ListElement: buf length not match") + return } - Self.buf = buf - Self.l = l - Self.part = part - return nil + //if l == 0 { + // logs.Warn("push zero") + //} + element = common.ListElementPool.Get() + element.Buf = buf + element.L = l + element.Part = part + return } type ReceiveWindowQueue struct { - chain *bufChain - length uint32 - stopOp chan struct{} - readOp chan struct{} - popWait uint32 - timeout time.Time + chain *bufChain + stopOp chan struct{} + readOp chan struct{} + lengthWait uint64 + timeout time.Time } func (Self *ReceiveWindowQueue) New() { @@ -151,45 +151,45 @@ func (Self *ReceiveWindowQueue) New() { Self.stopOp = make(chan struct{}, 2) } -func (Self *ReceiveWindowQueue) Push(element *ListElement) { +func (Self *ReceiveWindowQueue) Push(element *common.ListElement) { + var length, wait uint32 + for { + ptrs := atomic.LoadUint64(&Self.lengthWait) + length, wait = Self.chain.head.unpack(ptrs) + length += uint32(element.L) + if atomic.CompareAndSwapUint64(&Self.lengthWait, ptrs, Self.chain.head.pack(length, 0)) { + break + } + // another goroutine change the length or into wait, make sure + } + //logs.Warn("window push before", Self.Len(), uint32(element.l), len(element.buf)) Self.chain.pushHead(unsafe.Pointer(element)) - atomic.AddUint32(&Self.length, uint32(element.l)) - Self.allowPop() - return -} - -func (Self *ReceiveWindowQueue) pop() (element *ListElement) { - ptr, ok := Self.chain.popTail() - if ok { - element = (*ListElement)(ptr) - atomic.AddUint32(&Self.length, ^uint32(element.l-1)) - return + //logs.Warn("window push", Self.Len()) + if wait == 1 { + Self.allowPop() } return } -func (Self *ReceiveWindowQueue) Pop() (element *ListElement, err error) { - var iter bool +func (Self *ReceiveWindowQueue) Pop() (element *common.ListElement, err error) { + var length uint32 startPop: - element = Self.pop() - if element != nil { - return - } - if !iter { - iter = true - runtime.Gosched() - goto startPop - } - if atomic.CompareAndSwapUint32(&Self.popWait, 0, 1) { - iter = false + ptrs := atomic.LoadUint64(&Self.lengthWait) + length, _ = Self.chain.head.unpack(ptrs) + if length == 0 { + if !atomic.CompareAndSwapUint64(&Self.lengthWait, ptrs, Self.chain.head.pack(0, 1)) { + goto startPop // another goroutine is pushing + } t := Self.timeout.Sub(time.Now()) if t <= 0 { t = time.Minute } timer := time.NewTimer(t) defer timer.Stop() + //logs.Warn("queue into wait") select { case <-Self.readOp: + //logs.Warn("queue wait finish") goto startPop case <-Self.stopOp: err = io.EOF @@ -199,23 +199,34 @@ startPop: return } } - goto startPop + // length is not zero, so try to pop + for { + ptr, ok := Self.chain.popTail() + if ok { + //logs.Warn("window pop before", Self.Len()) + element = (*common.ListElement)(ptr) + atomic.AddUint64(&Self.lengthWait, ^(uint64(element.L)< Date: Sat, 9 Nov 2019 23:24:26 +0800 Subject: [PATCH 30/38] version --- lib/version/version.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/version/version.go b/lib/version/version.go index 4cc0532..1f14ef1 100644 --- a/lib/version/version.go +++ b/lib/version/version.go @@ -1,8 +1,8 @@ package version -const VERSION = "0.23.3" +const VERSION = "0.24.0" // Compulsory minimum version, Minimum downward compatibility to this version func GetVersion() string { - return "0.22.0" + return "0.24.0" } From bc1783cfb64c7e882eef7ac84580bb3bc7044971 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sun, 10 Nov 2019 21:04:35 +0800 Subject: [PATCH 31/38] 64bit alignment, readme --- README.md | 28 ++++++++++++++++++++++++++++ lib/mux/conn.go | 6 +++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f56c32a..4a9ddd4 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,7 @@ nps是一款轻量级、高性能、功能强大的**内网穿透**代理服务 * [获取用户真实ip](#获取用户真实ip) * [客户端地址显示](#客户端地址显示) * [客户端与服务端版本对比](#客户端与服务端版本对比) + * [Linux系统限制](#Linux系统限制) * [webAPI](#webAPI) * [贡献](#贡献) * [支持nps发展](#捐赠) @@ -144,6 +145,7 @@ nps是一款轻量级、高性能、功能强大的**内网穿透**代理服务 ```shell ./npc -server=1.1.1.1:8284 -vkey=客户端的密钥 ``` +**注意：运行服务端后，请确保能从客户端设备上正常访问配置文件中所配置的`bridge_port`端口，telnet，netcat这类的来检查** ### 域名解析 @@ -441,6 +443,27 @@ server_ip=xxx ``` ./npc -config=npc配置文件路径 ``` +可自行添加systemd service，例如：`npc.service` +``` +[Unit] +Description=npc - convenient proxy server client +Documentation=https://github.com/cnlh/nps/ +After=network-online.target remote-fs.target nss-lookup.target +Wants=network-online.target + +[Service] +Type=simple +KillMode=process +Restart=always +RestartSec=15s +StandardOutput=append:/var/log/nps/npc.log +ExecStartPre=/bin/echo 'Starting npc' +ExecStopPost=/bin/echo 'Stopping npc' +ExecStart=/absolutely path to/npc -server=ip:port -vkey=web界面中显示的密钥 + +[Install] +WantedBy=multi-user.target +``` #### 配置文件说明 [示例配置文件](https://github.com/cnlh/nps/tree/master/conf/npc.conf) ##### 全局配置 @@ -909,6 +932,11 @@ LevelInformational->6 LevelDebug->7 ### 客户端与服务端版本对比 为了程序正常运行，客户端与服务端的核心版本必须一致，否则将导致客户端无法成功连接致服务端。 +### Linux系统限制 +默认情况下linux对连接数量有限制，对于性能好的机器完全可以调整内核参数以处理更多的连接。 +`tcp_max_syn_backlog` `somaxconn` +酌情调整参数，增强网络性能 + ## webAPI ### webAPI验证说明 diff --git a/lib/mux/conn.go b/lib/mux/conn.go index cb982e8..7d2e351 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -144,11 +144,11 @@ func (s *conn) SetWriteDeadline(t time.Time) error { } type window struct { + remainingWait uint64 // 64bit alignment off uint32 maxSize uint32 closeOp bool closeOpCh chan struct{} - remainingWait uint64 mux *Mux } @@ -178,11 +178,11 @@ func (Self *window) CloseWindow() { } type ReceiveWindow struct { + window bufQueue ReceiveWindowQueue element *common.ListElement count int8 once sync.Once - window } func (Self *ReceiveWindow) New(mux *Mux) { @@ -344,10 +344,10 @@ func (Self *ReceiveWindow) CloseWindow() { } type SendWindow struct { + window buf []byte setSizeCh chan struct{} timeout time.Time - window } func (Self *SendWindow) New(mux *Mux) { From 2ca84c912b32e73ddae30916aa191caa973c3a52 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Wed, 13 Nov 2019 23:33:02 +0800 Subject: [PATCH 32/38] fix mux auto disconnect, add mux new connection queue --- lib/mux/conn.go | 8 ++-- lib/mux/mux.go | 81 +++++++++++++++++--------------- lib/mux/queue.go | 119 ++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 149 insertions(+), 59 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 7d2e351..23bc5d5 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -23,7 +23,7 @@ type conn struct { receiveWindow *ReceiveWindow sendWindow *SendWindow once sync.Once - label string + //label string } func NewConn(connId int32, mux *Mux, label ...string) *conn { @@ -36,9 +36,9 @@ func NewConn(connId int32, mux *Mux, label ...string) *conn { sendWindow: new(SendWindow), once: sync.Once{}, } - if len(label) > 0 { - c.label = label[0] - } + //if len(label) > 0 { + // c.label = label[0] + //} c.receiveWindow.New(mux) c.sendWindow.New(mux) //logm := &connLog{ diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 585c980..b64243f 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -14,21 +14,20 @@ import ( type Mux struct { net.Listener - conn net.Conn - connMap *connMap - newConnCh chan *conn - id int32 - closeChan chan struct{} - IsClose bool - pingOk int - latency float64 - bw *bandwidth - pingCh chan []byte - pingTimer *time.Timer - connType string - writeQueue PriorityQueue - //bufQueue BytesQueue - //sync.Mutex + conn net.Conn + connMap *connMap + newConnCh chan *conn + id int32 + closeChan chan struct{} + IsClose bool + pingOk int + latency float64 + bw *bandwidth + pingCh chan []byte + pingCheck bool + connType string + writeQueue PriorityQueue + newConnQueue ConnQueue } func NewMux(c net.Conn, connType string) *Mux { @@ -39,15 +38,14 @@ func NewMux(c net.Conn, connType string) *Mux { connMap: NewConnMap(), id: 0, closeChan: make(chan struct{}, 1), - newConnCh: make(chan *conn, 10), + newConnCh: make(chan *conn), bw: new(bandwidth), IsClose: false, connType: connType, pingCh: make(chan []byte), - pingTimer: time.NewTimer(15 * time.Second), } m.writeQueue.New() - //m.bufQueue.New() + m.newConnQueue.New() //read session by flag m.readSession() //ping @@ -101,6 +99,8 @@ func (s *Mux) sendInfo(flag uint8, id int32, data ...interface{}) { err = pack.NewPac(flag, id, data...) if err != nil { common.MuxPack.Put(pack) + logs.Error("mux: new pack err") + s.Close() return } s.writeQueue.Push(pack) @@ -124,7 +124,7 @@ func (s *Mux) packBuf() { err := pack.Pack(buffer) common.MuxPack.Put(pack) if err != nil { - logs.Warn("pack err", err) + logs.Error("mux: pack err", err) common.BuffPool.Put(buffer) break } @@ -134,7 +134,7 @@ func (s *Mux) packBuf() { n, err := buffer.WriteTo(s.conn) //common.BuffPool.Put(buffer) if err != nil || int(n) != l { - logs.Warn("close from write session fail ", err, n, l) + logs.Error("mux: close from write session fail ", err, n, l) s.Close() break } @@ -170,21 +170,23 @@ func (s *Mux) ping() { for { if s.IsClose { ticker.Stop() - if !s.pingTimer.Stop() { - <-s.pingTimer.C - } break } select { case <-ticker.C: } + if s.pingCheck { + logs.Error("mux: ping time out") + s.Close() + // more than 5 seconds not receive the ping return package, + // mux conn is damaged, maybe a packet drop, close it + break + } now, _ := time.Now().UTC().MarshalText() s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) - if !s.pingTimer.Stop() { - <-s.pingTimer.C - } - s.pingTimer.Reset(15 * time.Second) + s.pingCheck = true if s.pingOk > 10 && s.connType == "kcp" { + logs.Error("mux: kcp ping err") s.Close() break } @@ -203,12 +205,9 @@ func (s *Mux) pingReturn() { } select { case data = <-s.pingCh: + s.pingCheck = false case <-s.closeChan: break - case <-s.pingTimer.C: - logs.Error("mux: ping time out") - s.Close() - break } _ = now.UnmarshalText(data) latency := time.Now().UTC().Sub(now).Seconds() / 2 @@ -222,6 +221,15 @@ func (s *Mux) pingReturn() { } func (s *Mux) readSession() { + go func() { + var connection *conn + for { + connection = s.newConnQueue.Pop() + s.connMap.Set(connection.connId, connection) //it has been set before send ok + s.newConnCh <- connection + s.sendInfo(common.MUX_NEW_CONN_OK, connection.connId, nil) + } + }() go func() { pack := common.MuxPack.Get() var l uint16 @@ -233,18 +241,16 @@ func (s *Mux) readSession() { pack = common.MuxPack.Get() s.bw.StartRead() if l, err = pack.UnPack(s.conn); err != nil { + logs.Error("mux: read session unpack from connection err") + s.Close() break } s.bw.SetCopySize(l) s.pingOk = 0 switch pack.Flag { case common.MUX_NEW_CONN: //new connection - connection := NewConn(pack.Id, s, "npc ") - s.connMap.Set(pack.Id, connection) //it has been set before send ok - //go func(connection *conn) { - s.newConnCh <- connection - s.sendInfo(common.MUX_NEW_CONN_OK, connection.connId, nil) - //}(connection) + connection := NewConn(pack.Id, s) + s.newConnQueue.Push(connection) continue case common.MUX_PING_FLAG: //ping s.sendInfo(common.MUX_PING_RETURN, common.MUX_PING, pack.Content) @@ -261,6 +267,7 @@ func (s *Mux) readSession() { case common.MUX_NEW_MSG, common.MUX_NEW_MSG_PART: //new msg from remote connection err = s.newMsg(connection, pack) if err != nil { + logs.Error("mux: read session connection new msg err") connection.Close() } continue diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 4790779..0bfea18 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -33,6 +33,14 @@ func (Self *PriorityQueue) New() { } func (Self *PriorityQueue) Push(packager *common.MuxPackager) { + //logs.Warn("push start") + Self.push(packager) + Self.cond.Broadcast() + //logs.Warn("push finish") + return +} + +func (Self *PriorityQueue) push(packager *common.MuxPackager) { switch packager.Flag { case common.MUX_PING_FLAG, common.MUX_PING_RETURN: Self.highestChain.pushHead(unsafe.Pointer(packager)) @@ -44,8 +52,6 @@ func (Self *PriorityQueue) Push(packager *common.MuxPackager) { default: Self.lowestChain.pushHead(unsafe.Pointer(packager)) } - Self.cond.Signal() - return } const maxStarving uint8 = 8 @@ -121,6 +127,72 @@ func (Self *PriorityQueue) Stop() { Self.cond.Broadcast() } +type ConnQueue struct { + chain *bufChain + starving uint8 + stop bool + cond *sync.Cond +} + +func (Self *ConnQueue) New() { + Self.chain = new(bufChain) + Self.chain.new(32) + locker := new(sync.Mutex) + Self.cond = sync.NewCond(locker) +} + +func (Self *ConnQueue) Push(connection *conn) { + Self.chain.pushHead(unsafe.Pointer(connection)) + Self.cond.Broadcast() + return +} + +func (Self *ConnQueue) Pop() (connection *conn) { + var iter bool + for { + connection = Self.pop() + if connection != nil { + return + } + if Self.stop { + return + } + if iter { + break + // trying to pop twice + } + iter = true + runtime.Gosched() + } + Self.cond.L.Lock() + defer Self.cond.L.Unlock() + for connection = Self.pop(); connection == nil; { + if Self.stop { + return + } + //logs.Warn("queue into wait") + Self.cond.Wait() + // wait for it with no more iter + connection = Self.pop() + //logs.Warn("queue wait finish", packager) + } + return +} + +func (Self *ConnQueue) pop() (connection *conn) { + ptr, ok := Self.chain.popTail() + if ok { + connection = (*conn)(ptr) + return + } + return +} + +func (Self *ConnQueue) Stop() { + Self.stop = true + Self.cond.Broadcast() +} + func NewListElement(buf []byte, l uint16, part bool) (element *common.ListElement, err error) { if uint16(len(buf)) != l { err = errors.New("ListElement: buf length not match") @@ -180,24 +252,12 @@ startPop: if !atomic.CompareAndSwapUint64(&Self.lengthWait, ptrs, Self.chain.head.pack(0, 1)) { goto startPop // another goroutine is pushing } - t := Self.timeout.Sub(time.Now()) - if t <= 0 { - t = time.Minute - } - timer := time.NewTimer(t) - defer timer.Stop() - //logs.Warn("queue into wait") - select { - case <-Self.readOp: - //logs.Warn("queue wait finish") - goto startPop - case <-Self.stopOp: - err = io.EOF - return - case <-timer.C: - err = errors.New("mux.queue: read time out") + err = Self.waitPush() + // there is no more data in queue, wait for it + if err != nil { return } + goto startPop // wait finish, trying to get the new status } // length is not zero, so try to pop for { @@ -223,6 +283,29 @@ func (Self *ReceiveWindowQueue) allowPop() (closed bool) { } } +func (Self *ReceiveWindowQueue) waitPush() (err error) { + //logs.Warn("wait push") + //defer logs.Warn("wait push finish") + t := Self.timeout.Sub(time.Now()) + if t <= 0 { + t = time.Second * 10 + } + timer := time.NewTimer(t) + defer timer.Stop() + //logs.Warn("queue into wait") + select { + case <-Self.readOp: + //logs.Warn("queue wait finish") + return nil + case <-Self.stopOp: + err = io.EOF + return + case <-timer.C: + err = errors.New("mux.queue: read time out") + return + } +} + func (Self *ReceiveWindowQueue) Len() (n uint32) { ptrs := atomic.LoadUint64(&Self.lengthWait) n, _ = Self.chain.head.unpack(ptrs) From aaf79b21f03f04309b1bc10580e4943f2b6b32a0 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Tue, 19 Nov 2019 23:43:52 +0800 Subject: [PATCH 33/38] fine mux connection ping calculation, increase connection waiting time --- README.md | 13 +++++ lib/mux/conn.go | 6 ++- lib/mux/mux.go | 130 +++++++++++++++++++++++++++++++++++++++-------- lib/mux/queue.go | 2 +- 4 files changed, 127 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 4a9ddd4..8988d5a 100644 --- a/README.md +++ b/README.md @@ -824,6 +824,19 @@ nps支持对客户端的隧道数量进行限制，该功能默认是关闭的 nps主要通信默认基于多路复用，无需开启。 +多路复用基于TCP滑动窗口原理设计，动态计算延迟以及带宽来算出应该往网络管道中打入的流量。 +由于主要通信大多采用TCP协议，并无法探测其实时丢包情况，对于产生丢包重传的情况，采用较大的宽容度， +5分钟的等待时间，超时将会关闭当前隧道连接并重新建立，这将会抛弃当前所有的连接。 +在Linux上，可以通过调节内核参数来适应不同应用场景。 + +对于需求大带宽又有一定的丢包的场景，可以保持默认参数不变，尽可能少抛弃连接 +高并发下可根据[Linux系统限制](#Linux系统限制) 调整 + +对于延迟敏感而又有一定丢包的场景，可以适当调整TCP重传次数 +`tcp_syn_retries`, `tcp_retries1`, `tcp_retries2` +高并发同上 +nps会在系统主动关闭连接的时候拿到报错，进而重新建立隧道连接 + ### 环境变量渲染 npc支持环境变量渲染以适应在某些特殊场景下的要求。 diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 23bc5d5..7bb88ae 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -3,6 +3,7 @@ package mux import ( "errors" "io" + "math" "net" "runtime" "sync" @@ -208,7 +209,8 @@ func (Self *ReceiveWindow) calcSize() { // calculating maximum receive window size if Self.count == 0 { //logs.Warn("ping, bw", Self.mux.latency, Self.bw.Get()) - n := uint32(2 * Self.mux.latency * Self.mux.bw.Get() * 1.5 / float64(Self.mux.connMap.Size())) + n := uint32(2 * math.Float64frombits(atomic.LoadUint64(&Self.mux.latency)) * + Self.mux.bw.Get() * 1.5 / float64(Self.mux.connMap.Size())) if n < 8192 { n = 8192 } @@ -471,7 +473,7 @@ start: func (Self *SendWindow) waitReceiveWindow() (err error) { t := Self.timeout.Sub(time.Now()) if t < 0 { - t = time.Minute + t = time.Minute * 5 } timer := time.NewTimer(t) defer timer.Stop() diff --git a/lib/mux/mux.go b/lib/mux/mux.go index b64243f..3872f7e 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -13,21 +13,22 @@ import ( ) type Mux struct { + latency uint64 // we store latency in bits, but it's float64 net.Listener - conn net.Conn - connMap *connMap - newConnCh chan *conn - id int32 - closeChan chan struct{} - IsClose bool - pingOk int - latency float64 - bw *bandwidth - pingCh chan []byte - pingCheck bool - connType string - writeQueue PriorityQueue - newConnQueue ConnQueue + conn net.Conn + connMap *connMap + newConnCh chan *conn + id int32 + closeChan chan struct{} + IsClose bool + pingOk int + counter *latencyCounter + bw *bandwidth + pingCh chan []byte + pingCheckTime uint32 + connType string + writeQueue PriorityQueue + newConnQueue ConnQueue } func NewMux(c net.Conn, connType string) *Mux { @@ -43,6 +44,7 @@ func NewMux(c net.Conn, connType string) *Mux { IsClose: false, connType: connType, pingCh: make(chan []byte), + counter: newLatencyCounter(), } m.writeQueue.New() m.newConnQueue.New() @@ -175,16 +177,16 @@ func (s *Mux) ping() { select { case <-ticker.C: } - if s.pingCheck { + if atomic.LoadUint32(&s.pingCheckTime) >= 60 { logs.Error("mux: ping time out") s.Close() - // more than 5 seconds not receive the ping return package, + // more than 5 minutes not receive the ping return package, // mux conn is damaged, maybe a packet drop, close it break } now, _ := time.Now().UTC().MarshalText() s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) - s.pingCheck = true + atomic.AddUint32(&s.pingCheckTime, 1) if s.pingOk > 10 && s.connType == "kcp" { logs.Error("mux: kcp ping err") s.Close() @@ -205,16 +207,17 @@ func (s *Mux) pingReturn() { } select { case data = <-s.pingCh: - s.pingCheck = false + atomic.StoreUint32(&s.pingCheckTime, 0) case <-s.closeChan: break } _ = now.UnmarshalText(data) latency := time.Now().UTC().Sub(now).Seconds() / 2 - if latency < 0.5 && latency > 0 { - s.latency = latency + if latency > 0 { + atomic.StoreUint64(&s.latency, math.Float64bits(s.counter.Latency(latency))) + // convert float64 to bits, store it atomic } - //logs.Warn("latency", s.latency) + //logs.Warn("latency", math.Float64frombits(atomic.LoadUint64(&s.latency))) common.WindowBuff.Put(data) } }() @@ -379,3 +382,88 @@ func (Self *bandwidth) Get() (bw float64) { } return Self.readBandwidth } + +const counterBits = 4 +const counterMask = 1<> counterBits) & counterMask) + // we set head is 4 bits + min = uint8(idxs & counterMask) + return +} + +func (Self *latencyCounter) pack(head, min uint8) uint8 { + return uint8(head< value { + min = head + } + head++ + Self.headMin = Self.pack(head, min) +} + +func (Self *latencyCounter) minimal() (min uint8) { + var val float64 + var i uint8 + for i = 0; i < counterMask; i++ { + if Self.buf[i] > 0 { + if val > Self.buf[i] { + val = Self.buf[i] + min = i + } + } + } + return +} + +func (Self *latencyCounter) Latency(value float64) (latency float64) { + Self.add(value) + _, min := Self.unpack(Self.headMin) + latency = Self.buf[min] * Self.countSuccess() + return +} + +const lossRatio = 1.6 + +func (Self *latencyCounter) countSuccess() (successRate float64) { + var success, loss, i uint8 + _, min := Self.unpack(Self.headMin) + for i = 0; i < counterMask; i++ { + if Self.buf[i] > lossRatio*Self.buf[min] && Self.buf[i] > 0 { + loss++ + } + if Self.buf[i] <= lossRatio*Self.buf[min] && Self.buf[i] > 0 { + success++ + } + } + // counting all the data in the ring buf, except zero + successRate = float64(success) / float64(loss+success) + return +} diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 0bfea18..2fe8a44 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -288,7 +288,7 @@ func (Self *ReceiveWindowQueue) waitPush() (err error) { //defer logs.Warn("wait push finish") t := Self.timeout.Sub(time.Now()) if t <= 0 { - t = time.Second * 10 + t = time.Minute * 5 } timer := time.NewTimer(t) defer timer.Stop() From bfe08e5114e320fef84d67a6382a358f8254fe1f Mon Sep 17 00:00:00 2001 From: zhangzc <50092028+43280398@users.noreply.github.com> Date: Wed, 20 Nov 2019 11:47:55 +0800 Subject: [PATCH 34/38] Update util.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修改nginx代理转发后无法获取真实ip --- lib/common/util.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/common/util.go b/lib/common/util.go index 1f54a6f..9a60846 100755 --- a/lib/common/util.go +++ b/lib/common/util.go @@ -108,6 +108,9 @@ func ChangeHostAndHeader(r *http.Request, host string, header string, addr strin } } addr = strings.Split(addr, ":")[0] + if prior, ok := r.Header["X-Forwarded-For"]; ok { + addr = strings.Join(prior, ", ") + ", " + addr + } r.Header.Set("X-Forwarded-For", addr) r.Header.Set("X-Real-IP", addr) } From 9bb8230fc17e59101d397c16aef4562a481020a8 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Thu, 21 Nov 2019 23:53:06 +0800 Subject: [PATCH 35/38] fix several race condition, change slide window max size 2G to 32M, add buffer release --- lib/common/const.go | 3 ++- lib/mux/conn.go | 25 +++++++++++++++++- lib/mux/map.go | 63 +++++++++++++++++++++++---------------------- lib/mux/mux.go | 57 +++++++++++++++++++++++++++++++++------- lib/mux/queue.go | 40 ++++++++++++++++------------ 5 files changed, 129 insertions(+), 59 deletions(-) diff --git a/lib/common/const.go b/lib/common/const.go index f57ce4f..2fd5bb6 100644 --- a/lib/common/const.go +++ b/lib/common/const.go @@ -49,5 +49,6 @@ const ( MUX_PING_RETURN MUX_PING int32 = -1 MAXIMUM_SEGMENT_SIZE = PoolSizeWindow - MAXIMUM_WINDOW_SIZE = 1<<31 - 1 + MAXIMUM_WINDOW_SIZE = 1 << 25 // 1<<31-1 TCP slide window size is very large, + // we use 32M, reduce memory usage ) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 7bb88ae..c94ab4d 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -210,7 +210,7 @@ func (Self *ReceiveWindow) calcSize() { if Self.count == 0 { //logs.Warn("ping, bw", Self.mux.latency, Self.bw.Get()) n := uint32(2 * math.Float64frombits(atomic.LoadUint64(&Self.mux.latency)) * - Self.mux.bw.Get() * 1.5 / float64(Self.mux.connMap.Size())) + Self.mux.bw.Get() / float64(Self.mux.connMap.Size())) if n < 8192 { n = 8192 } @@ -279,6 +279,9 @@ copyData: // on the first Read method invoked, Self.off and Self.element.l // both zero value common.ListElementPool.Put(Self.element) + if Self.closeOp { + return 0, io.EOF + } Self.element, err = Self.bufQueue.Pop() // if the queue is empty, Pop method will wait until one element push // into the queue successful, or timeout. @@ -343,6 +346,26 @@ func (Self *ReceiveWindow) Stop() { func (Self *ReceiveWindow) CloseWindow() { Self.window.CloseWindow() Self.Stop() + Self.release() +} + +func (Self *ReceiveWindow) release() { + //if Self.element != nil { + // if Self.element.Buf != nil { + // common.WindowBuff.Put(Self.element.Buf) + // } + // common.ListElementPool.Put(Self.element) + //} + for { + Self.element = Self.bufQueue.TryPop() + if Self.element == nil { + return + } + if Self.element.Buf != nil { + common.WindowBuff.Put(Self.element.Buf) + } + common.ListElementPool.Put(Self.element) + } // release resource } type SendWindow struct { diff --git a/lib/mux/map.go b/lib/mux/map.go index 8f07dee..86d09b5 100644 --- a/lib/mux/map.go +++ b/lib/mux/map.go @@ -2,32 +2,35 @@ package mux import ( "sync" - "time" ) type connMap struct { connMap map[int32]*conn - closeCh chan struct{} + //closeCh chan struct{} sync.RWMutex } func NewConnMap() *connMap { connMap := &connMap{ connMap: make(map[int32]*conn), - closeCh: make(chan struct{}), + //closeCh: make(chan struct{}), } - go connMap.clean() + //go connMap.clean() return connMap } func (s *connMap) Size() (n int) { - return len(s.connMap) + s.Lock() + n = len(s.connMap) + s.Unlock() + return } func (s *connMap) Get(id int32) (*conn, bool) { s.Lock() - defer s.Unlock() - if v, ok := s.connMap[id]; ok && v != nil { + v, ok := s.connMap[id] + s.Unlock() + if ok && v != nil { return v, true } return nil, false @@ -35,40 +38,38 @@ func (s *connMap) Get(id int32) (*conn, bool) { func (s *connMap) Set(id int32, v *conn) { s.Lock() - defer s.Unlock() s.connMap[id] = v + s.Unlock() } func (s *connMap) Close() { - s.Lock() - defer s.Unlock() + //s.closeCh <- struct{}{} // stop the clean goroutine first for _, v := range s.connMap { - v.isClose = true + v.Close() // close all the connections in the mux } - s.closeCh <- struct{}{} } func (s *connMap) Delete(id int32) { s.Lock() - defer s.Unlock() delete(s.connMap, id) + s.Unlock() } -func (s *connMap) clean() { - ticker := time.NewTimer(time.Minute * 1) - for { - select { - case <-ticker.C: - s.Lock() - for _, v := range s.connMap { - if v.isClose { - delete(s.connMap, v.connId) - } - } - s.Unlock() - case <-s.closeCh: - ticker.Stop() - return - } - } -} +//func (s *connMap) clean() { +// ticker := time.NewTimer(time.Minute * 1) +// for { +// select { +// case <-ticker.C: +// s.Lock() +// for _, v := range s.connMap { +// if v.isClose { +// delete(s.connMap, v.connId) +// } +// } +// s.Unlock() +// case <-s.closeCh: +// ticker.Stop() +// return +// } +// } +//} diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 3872f7e..8c4febb 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -122,6 +122,9 @@ func (s *Mux) packBuf() { } buffer.Reset() pack := s.writeQueue.Pop() + if s.IsClose { + break + } //buffer := common.BuffPool.Get() err := pack.Pack(buffer) common.MuxPack.Put(pack) @@ -218,7 +221,9 @@ func (s *Mux) pingReturn() { // convert float64 to bits, store it atomic } //logs.Warn("latency", math.Float64frombits(atomic.LoadUint64(&s.latency))) - common.WindowBuff.Put(data) + if cap(data) > 0 { + common.WindowBuff.Put(data) + } } }() } @@ -227,7 +232,13 @@ func (s *Mux) readSession() { go func() { var connection *conn for { + if s.IsClose { + break + } connection = s.newConnQueue.Pop() + if s.IsClose { + break // make sure that is closed + } s.connMap.Set(connection.connId, connection) //it has been set before send ok s.newConnCh <- connection s.sendInfo(common.MUX_NEW_CONN_OK, connection.connId, nil) @@ -287,9 +298,9 @@ func (s *Mux) readSession() { connection.sendWindow.SetSize(pack.Window, pack.ReadLength) continue case common.MUX_CONN_CLOSE: //close the connection - s.connMap.Delete(pack.Id) - //go func(connection *conn) { connection.closeFlag = true + //s.connMap.Delete(pack.Id) + //go func(connection *conn) { connection.receiveWindow.Stop() // close signal to receive window //}(connection) continue @@ -322,17 +333,42 @@ func (s *Mux) newMsg(connection *conn, pack *common.MuxPackager) (err error) { return } -func (s *Mux) Close() error { +func (s *Mux) Close() (err error) { logs.Warn("close mux") if s.IsClose { return errors.New("the mux has closed") } s.IsClose = true s.connMap.Close() + s.connMap = nil //s.bufQueue.Stop() s.closeChan <- struct{}{} close(s.newConnCh) - return s.conn.Close() + err = s.conn.Close() + s.release() + return +} + +func (s *Mux) release() { + for { + pack := s.writeQueue.TryPop() + if pack == nil { + break + } + if pack.BasePackager.Content != nil { + common.WindowBuff.Put(pack.BasePackager.Content) + } + common.MuxPack.Put(pack) + } + for { + connection := s.newConnQueue.TryPop() + if connection == nil { + break + } + connection = nil + } + s.writeQueue.Stop() + s.newConnQueue.Stop() } //get new connId as unique flag @@ -352,7 +388,7 @@ type bandwidth struct { readStart time.Time lastReadStart time.Time bufLength uint16 - readBandwidth float64 + readBandwidth uint64 // store in bits, but it's float64 } func (Self *bandwidth) StartRead() { @@ -371,16 +407,17 @@ func (Self *bandwidth) SetCopySize(n uint16) { func (Self *bandwidth) calcBandWidth() { t := Self.readStart.Sub(Self.lastReadStart) - Self.readBandwidth = float64(Self.bufLength) / t.Seconds() + atomic.StoreUint64(&Self.readBandwidth, math.Float64bits(float64(Self.bufLength)/t.Seconds())) Self.bufLength = 0 } func (Self *bandwidth) Get() (bw float64) { // The zero value, 0 for numeric types - if Self.readBandwidth <= 0 { - Self.readBandwidth = 100 + bw = math.Float64frombits(atomic.LoadUint64(&Self.readBandwidth)) + if bw <= 0 { + bw = 100 } - return Self.readBandwidth + return } const counterBits = 4 diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 2fe8a44..2288e40 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -59,7 +59,7 @@ const maxStarving uint8 = 8 func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { var iter bool for { - packager = Self.pop() + packager = Self.TryPop() if packager != nil { return } @@ -75,20 +75,20 @@ func (Self *PriorityQueue) Pop() (packager *common.MuxPackager) { } Self.cond.L.Lock() defer Self.cond.L.Unlock() - for packager = Self.pop(); packager == nil; { + for packager = Self.TryPop(); packager == nil; { if Self.stop { return } //logs.Warn("queue into wait") Self.cond.Wait() // wait for it with no more iter - packager = Self.pop() + packager = Self.TryPop() //logs.Warn("queue wait finish", packager) } return } -func (Self *PriorityQueue) pop() (packager *common.MuxPackager) { +func (Self *PriorityQueue) TryPop() (packager *common.MuxPackager) { ptr, ok := Self.highestChain.popTail() if ok { packager = (*common.MuxPackager)(ptr) @@ -150,7 +150,7 @@ func (Self *ConnQueue) Push(connection *conn) { func (Self *ConnQueue) Pop() (connection *conn) { var iter bool for { - connection = Self.pop() + connection = Self.TryPop() if connection != nil { return } @@ -166,20 +166,20 @@ func (Self *ConnQueue) Pop() (connection *conn) { } Self.cond.L.Lock() defer Self.cond.L.Unlock() - for connection = Self.pop(); connection == nil; { + for connection = Self.TryPop(); connection == nil; { if Self.stop { return } //logs.Warn("queue into wait") Self.cond.Wait() // wait for it with no more iter - connection = Self.pop() + connection = Self.TryPop() //logs.Warn("queue wait finish", packager) } return } -func (Self *ConnQueue) pop() (connection *conn) { +func (Self *ConnQueue) TryPop() (connection *conn) { ptr, ok := Self.chain.popTail() if ok { connection = (*conn)(ptr) @@ -261,18 +261,26 @@ startPop: } // length is not zero, so try to pop for { - ptr, ok := Self.chain.popTail() - if ok { - //logs.Warn("window pop before", Self.Len()) - element = (*common.ListElement)(ptr) - atomic.AddUint64(&Self.lengthWait, ^(uint64(element.L)< Date: Sat, 23 Nov 2019 00:42:46 +0800 Subject: [PATCH 36/38] change some bandwidth calculation --- lib/common/pool.go | 2 +- lib/mux/conn.go | 22 +++++++++++++--------- lib/mux/mux.go | 32 +++++++++++++++++--------------- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/lib/common/pool.go b/lib/common/pool.go index 5d4da00..1f7a47e 100644 --- a/lib/common/pool.go +++ b/lib/common/pool.go @@ -10,7 +10,7 @@ const PoolSizeSmall = 100 const PoolSizeUdp = 1472 const PoolSizeCopy = 32 << 10 const PoolSizeBuffer = 4096 -const PoolSizeWindow = PoolSizeBuffer - 16 - 32 - 32 - 8 +const PoolSizeWindow = PoolSizeBuffer - 2 - 4 - 4 - 1 var BufPool = sync.Pool{ New: func() interface{} { diff --git a/lib/mux/conn.go b/lib/mux/conn.go index c94ab4d..1b7a920 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -190,7 +190,7 @@ func (Self *ReceiveWindow) New(mux *Mux) { // initial a window for receive Self.bufQueue.New() Self.element = common.ListElementPool.Get() - Self.maxSize = 4096 + Self.maxSize = common.MAXIMUM_SEGMENT_SIZE Self.mux = mux Self.window.New() } @@ -209,21 +209,25 @@ func (Self *ReceiveWindow) calcSize() { // calculating maximum receive window size if Self.count == 0 { //logs.Warn("ping, bw", Self.mux.latency, Self.bw.Get()) - n := uint32(2 * math.Float64frombits(atomic.LoadUint64(&Self.mux.latency)) * - Self.mux.bw.Get() / float64(Self.mux.connMap.Size())) - if n < 8192 { - n = 8192 + conns := Self.mux.connMap.Size() + n := uint32(math.Float64frombits(atomic.LoadUint64(&Self.mux.latency)) * + Self.mux.bw.Get() / float64(conns)) + if n < common.MAXIMUM_SEGMENT_SIZE*2 { + n = common.MAXIMUM_SEGMENT_SIZE * 2 } bufLen := Self.bufQueue.Len() if n < bufLen { n = bufLen } + if n < Self.maxSize/2 { + n = Self.maxSize / 2 + } // set the minimal size if n > 2*Self.maxSize { n = 2 * Self.maxSize } - if n > common.MAXIMUM_WINDOW_SIZE { - n = common.MAXIMUM_WINDOW_SIZE + if n > (common.MAXIMUM_WINDOW_SIZE / uint32(conns)) { + n = common.MAXIMUM_WINDOW_SIZE / uint32(conns) } // set the maximum size //logs.Warn("n", n) @@ -377,8 +381,8 @@ type SendWindow struct { func (Self *SendWindow) New(mux *Mux) { Self.setSizeCh = make(chan struct{}) - Self.maxSize = 4096 - atomic.AddUint64(&Self.remainingWait, uint64(4096)<= 16384 { + if Self.bufLength >= 3072000 { Self.lastReadStart, Self.readStart = Self.readStart, time.Now() Self.calcBandWidth() } } func (Self *bandwidth) SetCopySize(n uint16) { - Self.bufLength += n + Self.bufLength += uint32(n) } func (Self *bandwidth) calcBandWidth() { @@ -417,6 +418,7 @@ func (Self *bandwidth) Get() (bw float64) { if bw <= 0 { bw = 100 } + //logs.Warn(bw) return } From 32e3d411ad1d5287433e43becb9827e07324fcb9 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Sun, 24 Nov 2019 21:19:25 +0800 Subject: [PATCH 37/38] change initial window size --- lib/mux/conn.go | 10 +++++----- lib/mux/mux.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/mux/conn.go b/lib/mux/conn.go index 1b7a920..f665248 100644 --- a/lib/mux/conn.go +++ b/lib/mux/conn.go @@ -190,7 +190,7 @@ func (Self *ReceiveWindow) New(mux *Mux) { // initial a window for receive Self.bufQueue.New() Self.element = common.ListElementPool.Get() - Self.maxSize = common.MAXIMUM_SEGMENT_SIZE + Self.maxSize = common.MAXIMUM_SEGMENT_SIZE * 10 Self.mux = mux Self.window.New() } @@ -212,8 +212,8 @@ func (Self *ReceiveWindow) calcSize() { conns := Self.mux.connMap.Size() n := uint32(math.Float64frombits(atomic.LoadUint64(&Self.mux.latency)) * Self.mux.bw.Get() / float64(conns)) - if n < common.MAXIMUM_SEGMENT_SIZE*2 { - n = common.MAXIMUM_SEGMENT_SIZE * 2 + if n < common.MAXIMUM_SEGMENT_SIZE*10 { + n = common.MAXIMUM_SEGMENT_SIZE * 10 } bufLen := Self.bufQueue.Len() if n < bufLen { @@ -381,8 +381,8 @@ type SendWindow struct { func (Self *SendWindow) New(mux *Mux) { Self.setSizeCh = make(chan struct{}) - Self.maxSize = common.MAXIMUM_SEGMENT_SIZE - atomic.AddUint64(&Self.remainingWait, uint64(common.MAXIMUM_SEGMENT_SIZE)<= 3072000 { + if Self.bufLength >= common.MAXIMUM_SEGMENT_SIZE*300 { Self.lastReadStart, Self.readStart = Self.readStart, time.Now() Self.calcBandWidth() } From 78ebeba1bbd47ebf1f6cce7985d38b8e1041c5f9 Mon Sep 17 00:00:00 2001 From: ffdfgdfg Date: Wed, 27 Nov 2019 22:46:34 +0800 Subject: [PATCH 38/38] minor bug fix, docker --- Dockerfile.npc | 10 ++++++++++ Dockerfile.nps | 11 +++++++++++ README.md | 7 ++++++- lib/conn/listener.go | 7 +++++++ lib/mux/mux.go | 14 +++++++------- lib/mux/queue.go | 2 +- 6 files changed, 42 insertions(+), 9 deletions(-) create mode 100755 Dockerfile.npc create mode 100755 Dockerfile.nps diff --git a/Dockerfile.npc b/Dockerfile.npc new file mode 100755 index 0000000..ae6715a --- /dev/null +++ b/Dockerfile.npc @@ -0,0 +1,10 @@ +FROM golang as builder +WORKDIR /go/src/github.com/cnlh/nps +COPY . . +RUN go get -d -v ./... +RUN CGO_ENABLED=0 go build -ldflags="-w -s -extldflags -static" ./cmd/npc/npc.go + +FROM scratch +COPY --from=builder /go/src/github.com/cnlh/nps/npc / +VOLUME /conf +ENTRYPOINT ["/npc"] diff --git a/Dockerfile.nps b/Dockerfile.nps new file mode 100755 index 0000000..698ced9 --- /dev/null +++ b/Dockerfile.nps @@ -0,0 +1,11 @@ +FROM golang as builder +WORKDIR /go/src/github.com/cnlh/nps +COPY . . +RUN go get -d -v ./... +RUN CGO_ENABLED=0 go build -ldflags="-w -s -extldflags -static" ./cmd/nps/nps.go + +FROM scratch +COPY --from=builder /go/src/github.com/cnlh/nps/nps / +COPY --from=builder /go/src/github.com/cnlh/nps/web /web +VOLUME /conf +CMD ["/nps"] diff --git a/README.md b/README.md index 8988d5a..ba93cb5 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ nps是一款轻量级、高性能、功能强大的**内网穿透**代理服务 * [安装](#安装) * [编译安装](#源码安装) * [release安装](#release安装) + * [docker安装](#docker安装) * [使用示例（以web主控模式为主）](#使用示例) * [统一准备工作](#统一准备工作(必做)) * [http|https域名解析](#域名解析) @@ -121,7 +122,7 @@ nps是一款轻量级、高性能、功能强大的**内网穿透**代理服务 ## 安装 -### releases安装 +### release安装 > [releases](https://github.com/cnlh/nps/releases) 下载对应的系统版本即可，服务端和客户端是单独的 @@ -134,6 +135,10 @@ nps是一款轻量级、高性能、功能强大的**内网穿透**代理服务 > go build cmd/npc/npc.go +### docker安装 +> [server](https://hub.docker.com/r/ffdfgdfg/nps) +> [client](https://hub.docker.com/r/ffdfgdfg/npc) + ## 使用示例 ### 统一准备工作（必做） diff --git a/lib/conn/listener.go b/lib/conn/listener.go index f80e01d..bd8e443 100644 --- a/lib/conn/listener.go +++ b/lib/conn/listener.go @@ -43,9 +43,16 @@ func Accept(l net.Listener, f func(c net.Conn)) { if strings.Contains(err.Error(), "use of closed network connection") { break } + if strings.Contains(err.Error(), "the mux has closed") { + break + } logs.Warn(err) continue } + if c == nil { + logs.Warn("nil connection") + break + } go f(c) } } diff --git a/lib/mux/mux.go b/lib/mux/mux.go index 6c9c7fd..a43510a 100644 --- a/lib/mux/mux.go +++ b/lib/mux/mux.go @@ -21,7 +21,7 @@ type Mux struct { id int32 closeChan chan struct{} IsClose bool - pingOk int + pingOk uint32 counter *latencyCounter bw *bandwidth pingCh chan []byte @@ -101,7 +101,7 @@ func (s *Mux) sendInfo(flag uint8, id int32, data ...interface{}) { err = pack.NewPac(flag, id, data...) if err != nil { common.MuxPack.Put(pack) - logs.Error("mux: new pack err") + logs.Error("mux: new pack err", err) s.Close() return } @@ -191,12 +191,12 @@ func (s *Mux) ping() { now, _ := time.Now().UTC().MarshalText() s.sendInfo(common.MUX_PING_FLAG, common.MUX_PING, now) atomic.AddUint32(&s.pingCheckTime, 1) - if s.pingOk > 10 && s.connType == "kcp" { + if atomic.LoadUint32(&s.pingOk) > 10 && s.connType == "kcp" { logs.Error("mux: kcp ping err") s.Close() break } - s.pingOk++ + atomic.AddUint32(&s.pingOk, 1) } }() } @@ -256,12 +256,12 @@ func (s *Mux) readSession() { pack = common.MuxPack.Get() s.bw.StartRead() if l, err = pack.UnPack(s.conn); err != nil { - logs.Error("mux: read session unpack from connection err") + logs.Error("mux: read session unpack from connection err", err) s.Close() break } s.bw.SetCopySize(l) - s.pingOk = 0 + atomic.StoreUint32(&s.pingOk, 0) switch pack.Flag { case common.MUX_NEW_CONN: //new connection connection := NewConn(pack.Id, s) @@ -282,7 +282,7 @@ func (s *Mux) readSession() { case common.MUX_NEW_MSG, common.MUX_NEW_MSG_PART: //new msg from remote connection err = s.newMsg(connection, pack) if err != nil { - logs.Error("mux: read session connection new msg err") + logs.Error("mux: read session connection new msg err", err) connection.Close() } continue diff --git a/lib/mux/queue.go b/lib/mux/queue.go index 2288e40..212563c 100644 --- a/lib/mux/queue.go +++ b/lib/mux/queue.go @@ -209,10 +209,10 @@ func NewListElement(buf []byte, l uint16, part bool) (element *common.ListElemen } type ReceiveWindowQueue struct { + lengthWait uint64 chain *bufChain stopOp chan struct{} readOp chan struct{} - lengthWait uint64 timeout time.Time }