fix block problems

lib/mux/conn.go

@@ -17,22 +17,15 @@ type conn struct {
 	connStatusFailCh chan struct{}
 	readTimeOut      time.Time
 	writeTimeOut     time.Time
-	//readBuffer       []byte
-	//startRead        int //now read position
-	//endRead          int //now end read
-	//readFlag         bool
-	//readCh           chan struct{}
-	//readQueue        *sliceEntry
-	//stopWrite        bool
-	connId  int32
-	isClose bool
-	//readWait         bool
-	closeFlag     bool // close conn flag
-	hasWrite      int
-	receiveWindow *window
-	sendWindow    *window
-	mux           *Mux
-	once          sync.Once
+	connId           int32
+	isClose          bool
+	closeFlag        bool // close conn flag
+	receiveWindow    *window
+	sendWindow       *window
+	readCh           waitingCh
+	writeCh          waitingCh
+	mux              *Mux
+	once             sync.Once
 }
 
 func NewConn(connId int32, mux *Mux) *conn {
@@ -48,34 +41,32 @@ func NewConn(connId int32, mux *Mux) *conn {
 	}
 	c.receiveWindow.NewReceive()
 	c.sendWindow.NewSend()
+	c.readCh.new()
+	c.writeCh.new()
 	return c
 }
 
 func (s *conn) Read(buf []byte) (n int, err error) {
-	logs.Warn("starting conn read", s.connId)
+	//logs.Warn("starting conn read", s.connId)
 	if s.isClose || buf == nil {
 		return 0, errors.New("the conn has closed")
 	}
-	nCh := make(chan int)
-	errCh := make(chan error)
-	defer close(nCh)
-	defer close(errCh)
 	// waiting for takeout from receive window finish or timeout
-	go s.readWindow(buf, nCh, errCh)
+	go s.readWindow(buf, s.readCh.nCh, s.readCh.errCh)
 	if t := s.readTimeOut.Sub(time.Now()); t > 0 {
 		timer := time.NewTimer(t)
 		defer timer.Stop()
 		select {
 		case <-timer.C:
 			return 0, errors.New("read timeout")
-		case n = <-nCh:
-			err = <-errCh
+		case n = <-s.readCh.nCh:
+			err = <-s.readCh.errCh
 		}
 	} else {
-		n = <-nCh
-		err = <-errCh
+		n = <-s.readCh.nCh
+		err = <-s.readCh.errCh
 	}
-	logs.Warn("read window finish conn read n err buf", n, err, string(buf[:15]))
+	//logs.Warn("read window finish conn read n err buf", n, err, string(buf[:15]))
 	return
 }
 
@@ -97,21 +88,19 @@ func (s *conn) readWindow(buf []byte, nCh chan int, errCh chan error) {
 }
 
 func (s *conn) Write(buf []byte) (n int, err error) {
+	//logs.Warn("write starting", s.connId)
+	//defer logs.Warn("write end ", s.connId)
 	if s.isClose {
 		return 0, errors.New("the conn has closed")
 	}
 	if s.closeFlag {
-		logs.Warn("conn close by write ", s.connId)
+		//logs.Warn("conn close by write ", s.connId)
 		//s.Close()
 		return 0, errors.New("io: write on closed conn")
 	}
-
-	nCh := make(chan int)
-	errCh := make(chan error)
-	defer close(nCh)
-	defer close(errCh)
 	s.sendWindow.SetSendBuf(buf) // set the buf to send window
-	go s.write(nCh, errCh)
+	//logs.Warn("write set send buf success")
+	go s.write(s.writeCh.nCh, s.writeCh.errCh)
 	// waiting for send to other side or timeout
 	if t := s.writeTimeOut.Sub(time.Now()); t > 0 {
 		timer := time.NewTimer(t)
@@ -119,12 +108,12 @@ func (s *conn) Write(buf []byte) (n int, err error) {
 		select {
 		case <-timer.C:
 			return 0, errors.New("write timeout")
-		case n = <-nCh:
-			err = <-errCh
+		case n = <-s.writeCh.nCh:
+			err = <-s.writeCh.errCh
 		}
 	} else {
-		n = <-nCh
-		err = <-errCh
+		n = <-s.writeCh.nCh
+		err = <-s.writeCh.errCh
 	}
 	return
 }
@@ -160,7 +149,7 @@ func (s *conn) closeProcess() {
 	s.isClose = true
 	s.mux.connMap.Delete(s.connId)
 	if !s.mux.IsClose {
-		logs.Warn("conn send close", s.connId)
+		//logs.Warn("conn send close", s.connId)
 		// if server or user close the conn while reading, will get a io.EOF
 		// and this Close method will be invoke, send this signal to close other side
 		s.mux.sendInfo(common.MUX_CONN_CLOSE, s.connId, nil)
@@ -198,7 +187,6 @@ type window struct {
 	windowBuff          []byte
 	off                 uint16
 	readOp              chan struct{}
-	readWait            bool
 	WindowFull          bool
 	usableReceiveWindow chan uint16
 	WriteWg             sync.WaitGroup
@@ -213,13 +201,13 @@ func (Self *window) NewReceive() {
 	Self.windowBuff = common.WindowBuff.Get()
 	Self.readOp = make(chan struct{})
 	Self.WriteEndOp = make(chan struct{})
-	Self.closeOpCh = make(chan struct{}, 2)
+	Self.closeOpCh = make(chan struct{}, 3)
 }
 
 func (Self *window) NewSend() {
 	// initial a window for send
 	Self.usableReceiveWindow = make(chan uint16)
-	Self.closeOpCh = make(chan struct{}, 2)
+	Self.closeOpCh = make(chan struct{}, 3)
 }
 
 func (Self *window) SetSendBuf(buf []byte) {
@@ -269,38 +257,32 @@ func (Self *window) grow(n int) {
 
 func (Self *window) Write(p []byte) (n int, err error) {
 	if Self.closeOp {
-		logs.Warn("window write closed len p", len(p))
 		return 0, errors.New("conn.receiveWindow: write on closed window")
 	}
 	if len(p) > Self.Size() {
 		return 0, errors.New("conn.receiveWindow: write too large")
 	}
-	if Self.readWait {
-		defer Self.allowRead()
-	}
-	//logs.Warn("window write p string", len(p), string(p[:15]))
 	Self.mutex.Lock()
 	// slide the offset
 	if len(p) > Self.cap()-Self.len() {
 		// not enough space, need to allocate
 		Self.fullSlide()
-		//logs.Warn("window write full slide len cap", Self.len(), Self.cap())
 	} else {
 		// have enough space, re slice
 		Self.liteSlide()
-		//logs.Warn("window write lite slide len cap", Self.len(), Self.cap())
 	}
 	length := Self.len()                  // length before grow
 	Self.grow(len(p))                     // grow for copy
 	n = copy(Self.windowBuff[length:], p) // must copy data before allow Read
-	//logs.Warn("window write copy n len cap buf", n, Self.len(), Self.cap(), string(Self.windowBuff[Self.len()-n:Self.len()+15-n]))
+	if length == 0 {
+		// allow continue read
+		defer Self.allowRead()
+	}
 	Self.mutex.Unlock()
 	return n, nil
 }
 
 func (Self *window) allowRead() (closed bool) {
-	//logs.Warn("length 0 read op")
-	Self.readWait = false
 	if Self.closeOp {
 		close(Self.readOp)
 		return true
@@ -310,25 +292,25 @@ func (Self *window) allowRead() (closed bool) {
 		close(Self.readOp)
 		return true
 	case Self.readOp <- struct{}{}:
-		//logs.Warn("length 0 read op finish")
 		return false
 	}
 }
 
 func (Self *window) Read(p []byte) (n int, err error) {
-	logs.Warn("starting window read method len ", Self.len())
 	if Self.closeOp {
 		return 0, io.EOF // Write method receive close signal, returns eof
 	}
-	if Self.len() == 0 {
+	Self.mutex.Lock()
+	length := Self.len() // protect the length data, it invokes
+	// before Write lock and after Write unlock
+	Self.mutex.Unlock()
+	if length == 0 {
 		// window is empty, waiting for Write method send a success readOp signal
 		// or get timeout or close
-		Self.readWait = true
 		ticker := time.NewTicker(2 * time.Minute)
 		defer ticker.Stop()
 		select {
 		case _, ok := <-Self.readOp:
-			//logs.Warn("read window read op len cap", Self.len(), Self.cap())
 			if !ok {
 				return 0, errors.New("conn.receiveWindow: window closed")
 			}
@@ -341,19 +323,17 @@ func (Self *window) Read(p []byte) (n int, err error) {
 			return 0, io.EOF // receive close signal, returns eof
 		}
 	}
-	//logs.Warn("window read start len window buff", Self.len(), string(Self.windowBuff[Self.off:Self.off+15]))
 	Self.mutex.Lock()
 	n = copy(p, Self.windowBuff[Self.off:])
 	Self.off += uint16(n)
 	p = p[:n]
-	//logs.Warn("window read finish n len p p", n, len(p), string(p[:15]))
 	Self.mutex.Unlock()
 	return
 }
 
 func (Self *window) WriteTo() (p []byte, err error) {
 	if Self.closeOp {
-		logs.Warn("window write to closed")
+		//logs.Warn("window write to closed")
 		return nil, errors.New("conn.writeWindow: window closed")
 	}
 	if Self.len() == 0 {
@@ -373,6 +353,8 @@ waiting:
 		}
 	case <-ticker.C:
 		return nil, errors.New("conn.writeWindow: write to time out")
+	case <-Self.closeOpCh:
+		return nil, errors.New("conn.writeWindow: window closed")
 	}
 	if windowSize == 0 {
 		goto waiting // waiting for another usable window size
@@ -380,10 +362,8 @@ waiting:
 	Self.mutex.Lock()
 	if windowSize > uint16(Self.len()) {
 		// usable window size is bigger than window buff size, send the full buff
-		//logs.Warn("window size overflow windowSize len()", windowSize, Self.len())
 		windowSize = uint16(Self.len())
 	}
-	//logs.Warn("window buff off windowSize", Self.off, windowSize)
 	p = Self.windowBuff[Self.off : windowSize+Self.off]
 	Self.off += windowSize
 	Self.mutex.Unlock()
@@ -413,6 +393,22 @@ func (Self *window) CloseWindow() {
 	Self.closeOp = true
 	Self.closeOpCh <- struct{}{}
 	Self.closeOpCh <- struct{}{}
+	Self.closeOpCh <- struct{}{}
 	close(Self.closeOpCh)
 	return
 }
+
+type waitingCh struct {
+	nCh   chan int
+	errCh chan error
+}
+
+func (Self *waitingCh) new() {
+	Self.nCh = make(chan int)
+	Self.errCh = make(chan error)
+}
+
+func (Self *waitingCh) close() {
+	close(Self.nCh)
+	close(Self.errCh)
+}