From 49107d4733c73f85ea3fea56e7c5212fa9ef6f45 Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Sat, 9 Mar 2024 01:24:04 +1300 Subject: [PATCH 01/20] wip tcp --- transit/tcp/parser.go | 71 ++++++++++++++++++++++ transit/tcp/tcp-reader.go | 85 ++++++++++++++++++++++++++ transit/tcp/tcp-writer.go | 104 ++++++++++++++++++++++++++++++++ transit/tcp/tcp.go | 79 ++++++++++++++++++++++++ transit/tcp/udp.go | 124 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 463 insertions(+) create mode 100644 transit/tcp/parser.go create mode 100644 transit/tcp/tcp-reader.go create mode 100644 transit/tcp/tcp-writer.go create mode 100644 transit/tcp/tcp.go create mode 100644 transit/tcp/udp.go diff --git a/transit/tcp/parser.go b/transit/tcp/parser.go new file mode 100644 index 0000000..7ecc8fd --- /dev/null +++ b/transit/tcp/parser.go @@ -0,0 +1,71 @@ +package tcp + +import ( + "encoding/binary" + "errors" + "io" + "net" + + log "github.com/sirupsen/logrus" +) + +type Parser struct { + conn *net.TCPConn + maxPacketSize int + logger *log.Entry +} + +func NewParser(conn *net.TCPConn, maxPacketSize int, logger *log.Entry) *Parser { + return &Parser{ + conn: conn, + maxPacketSize: maxPacketSize, + logger: logger, + } +} + +const HeaderSize = 6 + +func (p *Parser) StartParsing(handler func(packetType byte, data []byte)) error { + buf := make([]byte, 0, 4096) // Initial buffer size, adjust based on needs + tmp := make([]byte, 1024) // Temporary buffer for reading from connection + + for { + n, err := p.conn.Read(tmp) + if err != nil { + if err != io.EOF { + return err // Handle error (excluding EOF) + } + break + } + + buf = append(buf, tmp[:n]...) + + for { + if len(buf) < HeaderSize { + break // Wait for more data + } + + if p.maxPacketSize > 0 && len(buf) > p.maxPacketSize { + return errors.New("incoming packet is larger than the 'maxPacketSize' limit") + } + + length := int(binary.BigEndian.Uint32(buf[1:5])) + if len(buf) < length { + break // Wait for complete packet + } + + crc := buf[1] ^ buf[2] ^ buf[3] ^ buf[4] ^ buf[5] + if crc != buf[0] { + return errors.New("invalid packet CRC") + } + + packetType := buf[5] + data := buf[HeaderSize:length] + handler(packetType, data) + + buf = buf[length:] // Move to next packet + } + } + + return nil +} diff --git a/transit/tcp/tcp-reader.go b/transit/tcp/tcp-reader.go new file mode 100644 index 0000000..90941af --- /dev/null +++ b/transit/tcp/tcp-reader.go @@ -0,0 +1,85 @@ +package tcp + +import ( + "bufio" + "fmt" + "net" + "sync" + + log "github.com/sirupsen/logrus" +) + +type TcpReader struct { + port int + listener net.Listener + sockets map[net.Conn]bool + logger *log.Entry + lock sync.Mutex +} + +func NewTcpReader(port int, logger *log.Entry) *TcpReader { + return &TcpReader{ + port: port, + sockets: make(map[net.Conn]bool), + logger: logger, + } +} + +func (r *TcpReader) Listen() { + var err error + r.listener, err = net.Listen("tcp", fmt.Sprintf(":%d", r.port)) + if err != nil { + r.logger.Fatal("Server error: ", err) + } + + r.logger.Info("TCP server is listening on port %d\n", r.port) + + go func() { + for { + conn, err := r.listener.Accept() + if err != nil { + r.logger.Error("Error accepting connection: ", err) + continue + } + r.lock.Lock() + r.sockets[conn] = true + r.lock.Unlock() + + go r.handleConnection(conn) + } + }() +} + +func (r *TcpReader) handleConnection(conn net.Conn) { + address := conn.RemoteAddr().String() + r.logger.Debug("New TCP client connected from '%s'\n", address) + + scanner := bufio.NewScanner(conn) + for scanner.Scan() { + msg := scanner.Text() + // TODO: Process incoming message here + // For example, emit to a channel or call a callback function + fmt.Printf("Received message from '%s': %s\n", address, msg) + // + } + + if err := scanner.Err(); err != nil { + r.logger.Error("Error reading from '%s': %s\n", address, err) + } + + r.closeSocket(conn) +} + +func (r *TcpReader) closeSocket(conn net.Conn) { + conn.Close() + r.lock.Lock() + delete(r.sockets, conn) + r.lock.Unlock() +} + +func (r *TcpReader) Close() { + r.listener.Close() + for conn := range r.sockets { + r.closeSocket(conn) + } +} diff --git a/transit/tcp/tcp-writer.go b/transit/tcp/tcp-writer.go new file mode 100644 index 0000000..7e564a3 --- /dev/null +++ b/transit/tcp/tcp-writer.go @@ -0,0 +1,104 @@ +package tcp + +import ( + "encoding/binary" + "errors" + "fmt" + "net" + "sync" + "time" + + log "github.com/sirupsen/logrus" +) + +type TcpWriter struct { + sockets map[string]*net.TCPConn + opts WriterOptions + logger *log.Entry + lock sync.Mutex +} + +type WriterOptions struct { + MaxConnections int +} + +func NewTcpWriter(opts WriterOptions, logger *log.Entry) *TcpWriter { + return &TcpWriter{ + sockets: make(map[string]*net.TCPConn), + opts: opts, + logger: logger, + } +} + +func (w *TcpWriter) Connect(nodeID, host string, port int) (*net.TCPConn, error) { + w.lock.Lock() + defer w.lock.Unlock() + + if socket, exists := w.sockets[nodeID]; exists && socket != nil { + return socket, nil + } + + addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", host, port)) + if err != nil { + return nil, err + } + + conn, err := net.DialTCP("tcp", nil, addr) + if err != nil { + return nil, err + } + + conn.SetNoDelay(true) + conn.SetKeepAlive(true) + conn.SetKeepAlivePeriod(3 * time.Minute) + + w.sockets[nodeID] = conn + return conn, nil +} + +func (w *TcpWriter) Send(nodeID string, packetType byte, data []byte) error { + socket, exists := w.sockets[nodeID] + if !exists || socket == nil { + return errors.New("connection does not exist") + } + + header := make([]byte, 6) + binary.BigEndian.PutUint32(header[1:], uint32(len(data)+len(header))) + header[5] = packetType + crc := header[1] ^ header[2] ^ header[3] ^ header[4] ^ header[5] + header[0] = crc + + payload := append(header, data...) + _, err := socket.Write(payload) + return err +} + +func (w *TcpWriter) manageConnections() { + // Simplified version: Close excess connections + w.lock.Lock() + defer w.lock.Unlock() + + if len(w.sockets) <= w.opts.MaxConnections { + return + } + + for nodeID, socket := range w.sockets { + socket.Close() + delete(w.sockets, nodeID) + w.logger.Debug("Closed connection to node %s", nodeID) + + if len(w.sockets) <= w.opts.MaxConnections { + break + } + } +} + +func (w *TcpWriter) Close() { + w.lock.Lock() + defer w.lock.Unlock() + + for nodeID, socket := range w.sockets { + socket.Close() + delete(w.sockets, nodeID) + } +} diff --git a/transit/tcp/tcp.go b/transit/tcp/tcp.go new file mode 100644 index 0000000..95ef64d --- /dev/null +++ b/transit/tcp/tcp.go @@ -0,0 +1,79 @@ +package tcp + +import ( + "time" + + log "github.com/sirupsen/logrus" +) + +type TcpTransporter struct { + opts TransporterOptions + reader *TcpReader // Placeholder for actual implementation + writer *TcpWriter // Placeholder for actual implementation + udpServer *UdpServer // Placeholder for actual implementation + gossipTimer *time.Ticker + logger *log.Entry +} + +type TransporterOptions struct { + UdpDiscovery bool + UdpPort int + UdpBindAddress string + UdpPeriod time.Duration + UdpReuseAddr bool + UdpMaxDiscovery int + UdpMulticast string + UdpMulticastTTL int + UdpBroadcast bool + Port int + Urls []string + UseHostname bool + GossipPeriod time.Duration + MaxConnections int + MaxPacketSize int + Logger *log.Entry +} + +func NewTcpTransporter(options TransporterOptions) *TcpTransporter { + return &TcpTransporter{ + opts: options, + logger: options.Logger, + } +} + +func (t *TcpTransporter) Connect() error { + // Example: Starting TCP and UDP server in Go + if err := t.startTcpServer(); err != nil { + return err + } + if t.opts.UdpDiscovery { + if err := t.startUdpServer(); err != nil { + return err + } + } + t.startTimers() + t.logger.Info("TCP Transporter started.") + // Additional setup and connection logic goes here + return nil +} + +// Placeholder for the actual TCP and UDP start methods +func (t *TcpTransporter) startTcpServer() error { + // TCP server setup and start logic + return nil +} + +func (t *TcpTransporter) startUdpServer() error { + // UDP server setup and start logic + return nil +} + +func (t *TcpTransporter) startTimers() { + // Example: Starting a ticker for gossip protocol + t.gossipTimer = time.NewTicker(t.opts.GossipPeriod * time.Second) + go func() { + for range t.gossipTimer.C { + // Handle gossip timer tick + } + }() +} diff --git a/transit/tcp/udp.go b/transit/tcp/udp.go new file mode 100644 index 0000000..10b2317 --- /dev/null +++ b/transit/tcp/udp.go @@ -0,0 +1,124 @@ +package tcp + +import ( + "fmt" + "net" + "time" + + "golang.org/x/net/ipv4" + + log "github.com/sirupsen/logrus" +) + +type UdpServer struct { + conn *net.UDPConn + opts UdpServerOptions + logger *log.Entry + discoverTimer *time.Ticker +} + +type UdpServerOptions struct { + Port int + Multicast string + MulticastTTL int + Broadcast bool + BroadcastAddrs []string // Optional, specify if not broadcasting to default subnet broadcast addresses + DiscoverPeriod time.Duration +} + +func NewUdpServer(opts UdpServerOptions, logger *log.Entry) *UdpServer { + return &UdpServer{ + opts: opts, + logger: logger, + } +} + +func (u *UdpServer) Bind() error { + addr := net.UDPAddr{ + Port: u.opts.Port, + IP: net.ParseIP("0.0.0.0"), + } + conn, err := net.ListenUDP("udp", &addr) + if err != nil { + return err + } + u.conn = conn + + if u.opts.Multicast != "" { + err := u.joinMulticastGroup() + if err != nil { + u.logger.Println("Error joining multicast group:", err) + // Handle non-fatal error, as we can continue in broadcast mode + } + } + + go u.handleIncomingMessages() + + u.startDiscovering() + + return nil +} + +func (u *UdpServer) handleIncomingMessages() { + buffer := make([]byte, 2048) + for { + n, addr, err := u.conn.ReadFromUDP(buffer) + if err != nil { + u.logger.Println("Error reading from UDP:", err) + continue + } + message := string(buffer[:n]) + u.logger.Printf("Received message from %s: %s", addr.String(), message) + // Process message here + } +} + +func (u *UdpServer) startDiscovering() { + u.discoverTimer = time.NewTicker(u.opts.DiscoverPeriod) + go func() { + for range u.discoverTimer.C { + u.broadcastDiscoveryMessage() + } + }() +} + +func (u *UdpServer) broadcastDiscoveryMessage() { + message := []byte("discovery message") // Customize your message + destAddr := &net.UDPAddr{IP: net.IPv4bcast, Port: u.opts.Port} + if _, err := u.conn.WriteToUDP(message, destAddr); err != nil { + u.logger.Println("Error broadcasting discovery message:", err) + } +} + +func (u *UdpServer) StopDiscovering() { + if u.discoverTimer != nil { + u.discoverTimer.Stop() + u.discoverTimer = nil + } +} + +func (u *UdpServer) Close() { + u.StopDiscovering() + if u.conn != nil { + u.conn.Close() + } +} + +func (u *UdpServer) joinMulticastGroup() error { + multicastAddr := net.ParseIP(u.opts.Multicast) + if multicastAddr == nil { + return fmt.Errorf("invalid multicast address: %s", u.opts.Multicast) + } + iface, err := net.InterfaceByName("eth0") // Specify the appropriate interface, or iterate over all if needed + if err != nil { + return fmt.Errorf("failed to get interface: %v", err) + } + p := ipv4.NewPacketConn(u.conn) + if err := p.JoinGroup(iface, &net.UDPAddr{IP: multicastAddr}); err != nil { + return fmt.Errorf("failed to join multicast group: %v", err) + } + if err := p.SetMulticastLoopback(true); err != nil { + return fmt.Errorf("failed to set multicast loopback: %v", err) + } + return nil +} From d06ad1e3233d2ad157538ad70e489c735e4ae4f7 Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Thu, 28 Mar 2024 15:51:04 +1300 Subject: [PATCH 02/20] handle receiving messages --- transit/pubsub/pubsub.go | 54 ++++++- transit/tcp/gossip.go | 15 ++ transit/tcp/tcp-reader.go | 110 ++++++++++---- transit/tcp/tcp-transporter.go | 192 +++++++++++++++++++++++++ transit/{tcp => tcp_gpt}/parser.go | 0 transit/tcp_gpt/tcp-reader.go | 85 +++++++++++ transit/{tcp => tcp_gpt}/tcp-writer.go | 0 transit/{tcp => tcp_gpt}/tcp.go | 0 transit/{tcp => tcp_gpt}/udp.go | 31 +++- 9 files changed, 457 insertions(+), 30 deletions(-) create mode 100644 transit/tcp/gossip.go create mode 100644 transit/tcp/tcp-transporter.go rename transit/{tcp => tcp_gpt}/parser.go (100%) create mode 100644 transit/tcp_gpt/tcp-reader.go rename transit/{tcp => tcp_gpt}/tcp-writer.go (100%) rename transit/{tcp => tcp_gpt}/tcp.go (100%) rename transit/{tcp => tcp_gpt}/udp.go (75%) diff --git a/transit/pubsub/pubsub.go b/transit/pubsub/pubsub.go index ecd7134..c25fc92 100644 --- a/transit/pubsub/pubsub.go +++ b/transit/pubsub/pubsub.go @@ -18,6 +18,7 @@ import ( "github.com/moleculer-go/moleculer/transit/kafka" "github.com/moleculer-go/moleculer/transit/memory" "github.com/moleculer-go/moleculer/transit/nats" + "github.com/moleculer-go/moleculer/transit/tcp" "github.com/moleculer-go/moleculer/util" "github.com/moleculer-go/moleculer" @@ -170,6 +171,9 @@ func (pubsub *PubSub) createTransport() transit.Transport { } else if pubsub.broker.Config.Transporter == "STAN" { pubsub.logger.Info("Transporter: NatsStreamingTransporter") transport = pubsub.createStanTransporter() + } else if pubsub.broker.Config.Transporter == "TCP" { + pubsub.logger.Info("Transporter: TCP") + transport = pubsub.createTCPTransporter() } else if isNats(pubsub.broker.Config.Transporter) { pubsub.logger.Info("Transporter: NatsTransporter") transport = pubsub.createNatsTransporter() @@ -213,7 +217,6 @@ func (pubsub *PubSub) createKafkaTransporter() transit.Transport { func (pubsub *PubSub) createNatsTransporter() transit.Transport { pubsub.logger.Debug("createNatsTransporter()") - return nats.CreateNatsTransporter(nats.NATSOptions{ URL: pubsub.broker.Config.Transporter, Name: pubsub.broker.LocalNode().GetID(), @@ -225,6 +228,51 @@ func (pubsub *PubSub) createNatsTransporter() transit.Transport { }) } +func (pubsub *PubSub) createTCPTransporter() transit.Transport { + pubsub.logger.Debug("createTCPTransporter()") + tcpTransporter := tcp.CreateTCPTransporter(tcp.TCPOptions{ + // Enable UDP discovery + UdpDiscovery: true, + // Reusing UDP server socket + UdpReuseAddr: true, + + // UDP port + UdpPort: 4445, + // UDP bind address (if null, bind on all interfaces) + UdpBindAddress: "", + // UDP sending period (seconds) + UdpPeriod: 30, + + // Multicast address. + UdpMulticast: "239.0.0.0", + // Multicast TTL setting + UdpMulticastTTL: 1, + + // Send broadcast (Boolean, String, Array) + UdpBroadcast: false, + + // TCP server port. 0 means random port + Port: 0, + // Static remote nodes address list (when UDP discovery is not available) + Urls: []string{}, + // Use hostname as preffered connection address + UseHostname: true, + + // Gossip sending period in seconds + GossipPeriod: 2, + // Maximum enabled outgoing connections. If reach, close the old connections + MaxConnections: 32, + // Maximum TCP packet size + MaxPacketSize: 1 * 1024 * 1024, + + NodeId: pubsub.broker.LocalNode().GetID(), + Logger: pubsub.logger.WithField("transport", "tcp"), + Serializer: pubsub.serializer, + }) + var transport transit.Transport = &tcpTransporter + return transport +} + func (pubsub *PubSub) createStanTransporter() transit.Transport { broker := pubsub.broker logger := broker.Logger("transport", "stan") @@ -285,7 +333,7 @@ func (pubsub *PubSub) waitForNeighbours() bool { } } -//DiscoverNodes will check if there are neighbours and return true if any are found ;). +// DiscoverNodes will check if there are neighbours and return true if any are found ;). func (pubsub *PubSub) DiscoverNodes() chan bool { result := make(chan bool) go func() { @@ -552,7 +600,7 @@ func (pubsub *PubSub) requestHandler() transit.TransportHandler { } } -//eventHandler handles when a event msg is sent to this broker +// eventHandler handles when a event msg is sent to this broker func (pubsub *PubSub) eventHandler() transit.TransportHandler { return func(message moleculer.Payload) { values := pubsub.serializer.PayloadToContextMap(message) diff --git a/transit/tcp/gossip.go b/transit/tcp/gossip.go new file mode 100644 index 0000000..1915af9 --- /dev/null +++ b/transit/tcp/gossip.go @@ -0,0 +1,15 @@ +package tcp + +type Gossip struct { +} + +func (g *Gossip) processHello(msgBytes *[]byte) { +} + +// processRequest +func (g *Gossip) processRequest(msgBytes *[]byte) { +} + +// processResponse +func (g *Gossip) processResponse(msgBytes *[]byte) { +} diff --git a/transit/tcp/tcp-reader.go b/transit/tcp/tcp-reader.go index 90941af..88db887 100644 --- a/transit/tcp/tcp-reader.go +++ b/transit/tcp/tcp-reader.go @@ -1,7 +1,7 @@ package tcp import ( - "bufio" + "encoding/binary" "fmt" "net" "sync" @@ -9,19 +9,32 @@ import ( log "github.com/sirupsen/logrus" ) +type State int + +const ( + LISTENING State = iota + CLOSED +) + +type OnMessageFunc func(msgType int, msgBytes *[]byte) + type TcpReader struct { - port int - listener net.Listener - sockets map[net.Conn]bool - logger *log.Entry - lock sync.Mutex + port int + listener net.Listener + sockets map[net.Conn]bool + logger *log.Entry + lock sync.Mutex + state State + maxPacketSize int + onMessage OnMessageFunc } -func NewTcpReader(port int, logger *log.Entry) *TcpReader { +func NewTcpReader(port int, onMessage OnMessageFunc, logger *log.Entry) *TcpReader { return &TcpReader{ - port: port, - sockets: make(map[net.Conn]bool), - logger: logger, + port: port, + sockets: make(map[net.Conn]bool), + logger: logger, + onMessage: onMessage, } } @@ -32,10 +45,11 @@ func (r *TcpReader) Listen() { r.logger.Fatal("Server error: ", err) } - r.logger.Info("TCP server is listening on port %d\n", r.port) + r.logger.Infof("TCP server is listening on port %d", r.port) + r.state = LISTENING go func() { - for { + for r.state == LISTENING { conn, err := r.listener.Accept() if err != nil { r.logger.Error("Error accepting connection: ", err) @@ -52,24 +66,71 @@ func (r *TcpReader) Listen() { func (r *TcpReader) handleConnection(conn net.Conn) { address := conn.RemoteAddr().String() - r.logger.Debug("New TCP client connected from '%s'\n", address) - - scanner := bufio.NewScanner(conn) - for scanner.Scan() { - msg := scanner.Text() - // TODO: Process incoming message here - // For example, emit to a channel or call a callback function - fmt.Printf("Received message from '%s': %s\n", address, msg) - // - } + r.logger.Debugf("New TCP client connected from '%s'\n", address) + + var err error - if err := scanner.Err(); err != nil { - r.logger.Error("Error reading from '%s': %s\n", address, err) + for err == nil { + msgType, msgBytes, e := r.readMessage(conn) + err = e + if err == nil { + r.logger.Errorf("Error reading message from '%s': %s", address, err) + break + } + r.onMessage(msgType, &msgBytes) } r.closeSocket(conn) } +func (r *TcpReader) readMessage(conn net.Conn) (msgType int, msg []byte, err error) { + var buf []byte + + for { + // Read data from the connection + chunk := make([]byte, 256) + n, err := conn.Read(chunk) + if err != nil { + return 0, nil, err + } + chunk = chunk[:n] + + // If there's a previous chunk, concatenate them + if buf != nil { + buf = append(buf, chunk...) + } else { + buf = chunk + } + + // If the buffer is too short, wait for the next chunk + if len(buf) < 6 { + continue + } + + // If the buffer is larger than the max packet size, return an error + if r.maxPacketSize > 0 && len(buf) > r.maxPacketSize { + return 0, nil, fmt.Errorf("Incoming packet is larger than the 'maxPacketSize' limit (%d > %d)!", len(buf), r.maxPacketSize) + } + + // Check the CRC + crc := buf[1] ^ buf[2] ^ buf[3] ^ buf[4] ^ buf[5] + if crc != buf[0] { + return 0, nil, fmt.Errorf("Invalid packet CRC! %d", crc) + } + + length := int(binary.BigEndian.Uint32(buf[1:])) + + // If the buffer contains a complete message, return it + if len(buf) >= length { + msg = buf[6:length] + msgType = int(buf[5]) // You'll need to replace this with your actual resolvePacketType function + return msgType, msg, nil + } + + // If the buffer doesn't contain a complete message, wait for the next chunk + } +} + func (r *TcpReader) closeSocket(conn net.Conn) { conn.Close() r.lock.Lock() @@ -78,6 +139,7 @@ func (r *TcpReader) closeSocket(conn net.Conn) { } func (r *TcpReader) Close() { + r.state = CLOSED r.listener.Close() for conn := range r.sockets { r.closeSocket(conn) diff --git a/transit/tcp/tcp-transporter.go b/transit/tcp/tcp-transporter.go new file mode 100644 index 0000000..2f29e43 --- /dev/null +++ b/transit/tcp/tcp-transporter.go @@ -0,0 +1,192 @@ +package tcp + +import ( + "github.com/moleculer-go/moleculer" + "github.com/moleculer-go/moleculer/serializer" + "github.com/moleculer-go/moleculer/transit" + + log "github.com/sirupsen/logrus" +) + +type TCPTransporter struct { + options TCPOptions + tcpReader *TcpReader + gossip *Gossip + + validateMsg transit.ValidateMsgFunc + serializer serializer.Serializer + handlers map[string][]transit.TransportHandler +} + +type TCPOptions struct { + + // Enable UDP discovery + UdpDiscovery bool + // Reusing UDP server socket + UdpReuseAddr bool + + // UDP port + UdpPort int + // UDP bind address (if null, bind on all interfaces) + UdpBindAddress string + // UDP sending period (seconds) + UdpPeriod int + + // Multicast address. + UdpMulticast string + // Multicast TTL setting + UdpMulticastTTL int + + // Send broadcast (Boolean, String, Array) + UdpBroadcast bool + + // TCP server port. Null or 0 means random port + Port int + // Static remote nodes address list (when UDP discovery is not available) + Urls []string + // Use hostname as preffered connection address + UseHostname bool + + // Gossip sending period in seconds + GossipPeriod int + // Maximum enabled outgoing connections. If reach, close the old connections + MaxConnections int + // Maximum TCP packet size + MaxPacketSize int + + Prefix string + NodeId string + Logger *log.Entry + Serializer serializer.Serializer + ValidateMsg transit.ValidateMsgFunc +} + +func CreateTCPTransporter(options TCPOptions) TCPTransporter { + transport := TCPTransporter{options: options} + return transport +} + +func (transporter *TCPTransporter) Connect() chan error { + transporter.options.Logger.Info("TCP Transported Connect()") + endChan := make(chan error) + go func() { + transporter.startTcpServer() + transporter.startUDPServer() + transporter.startGossip() + endChan <- nil + }() + return endChan +} + +type MessageType int + +const ( + PACKET_EVENT = 1 + PACKET_REQUEST = 2 + PACKET_RESPONSE = 3 + PACKET_PING = 4 + PACKET_PONG = 5 + PACKET_GOSSIP_REQ = 6 + PACKET_GOSSIP_RES = 7 + PACKET_GOSSIP_HELLO = 8 +) + +func (transporter *TCPTransporter) onTcpMessage(msgType int, msgBytes *[]byte) { + switch msgType { + case PACKET_GOSSIP_HELLO: + transporter.gossip.processHello(msgBytes) + case PACKET_GOSSIP_REQ: + transporter.gossip.processRequest(msgBytes) + case PACKET_GOSSIP_RES: + transporter.gossip.processResponse(msgBytes) + default: + transporter.incomingMessage(msgType, msgBytes) + } +} + +func (transporter *TCPTransporter) msgTypeToCommand(msgType int) string { + switch msgType { + case PACKET_EVENT: + return "EVENT" + case PACKET_REQUEST: + return "REQ" + case PACKET_RESPONSE: + return "RES" + // case PACKET_DISCOVER: + // return "DISCOVER" + // case PACKET_INFO: + // return "INFO" + // case PACKET_DISCONNECT: + // return "DISCONNECT" + // case PACKET_HEARTBEAT: + // return "HEARTBEAT" + case PACKET_PING: + return "PING" + case PACKET_PONG: + return "PONG" + case PACKET_GOSSIP_REQ: + return "GOSSIP_REQ" + case PACKET_GOSSIP_RES: + return "GOSSIP_RES" + case PACKET_GOSSIP_HELLO: + return "GOSSIP_HELLO" + default: + return "???" + } +} + +func (transporter *TCPTransporter) incomingMessage(msgType int, msgBytes *[]byte) { + command := transporter.msgTypeToCommand(msgType) + message := transporter.serializer.BytesToPayload(msgBytes) + if transporter.validateMsg(message) { + if handlers, ok := transporter.handlers[command]; ok { + for _, handler := range handlers { + handler(message) + } + } + } +} + +func (transporter *TCPTransporter) startTcpServer() { + transporter.tcpReader = NewTcpReader(transporter.options.Port, transporter.onTcpMessage, transporter.options.Logger) +} + +func (transporter *TCPTransporter) startUDPServer() { + +} + +func (transporter *TCPTransporter) startGossip() { + transporter.gossip = &Gossip{} +} + +func (transporter *TCPTransporter) Disconnect() chan error { + endChan := make(chan error) + go func() { + // Additional disconnection logic goes here + endChan <- nil + }() + return endChan +} + +func (transporter *TCPTransporter) Subscribe(command, nodeID string, handler transit.TransportHandler) { + if _, ok := transporter.handlers[command]; !ok { + transporter.handlers[command] = make([]transit.TransportHandler, 0) + } + transporter.handlers[command] = append(transporter.handlers[command], handler) +} + +func (transporter *TCPTransporter) Publish(command, nodeID string, message moleculer.Payload) { + // Additional publish logic goes here +} + +func (transporter *TCPTransporter) SetPrefix(prefix string) { + transporter.options.Prefix = prefix +} + +func (transporter *TCPTransporter) SetNodeID(nodeID string) { + transporter.options.NodeId = nodeID +} + +func (transporter *TCPTransporter) SetSerializer(serializer serializer.Serializer) { + transporter.options.Serializer = serializer +} diff --git a/transit/tcp/parser.go b/transit/tcp_gpt/parser.go similarity index 100% rename from transit/tcp/parser.go rename to transit/tcp_gpt/parser.go diff --git a/transit/tcp_gpt/tcp-reader.go b/transit/tcp_gpt/tcp-reader.go new file mode 100644 index 0000000..90941af --- /dev/null +++ b/transit/tcp_gpt/tcp-reader.go @@ -0,0 +1,85 @@ +package tcp + +import ( + "bufio" + "fmt" + "net" + "sync" + + log "github.com/sirupsen/logrus" +) + +type TcpReader struct { + port int + listener net.Listener + sockets map[net.Conn]bool + logger *log.Entry + lock sync.Mutex +} + +func NewTcpReader(port int, logger *log.Entry) *TcpReader { + return &TcpReader{ + port: port, + sockets: make(map[net.Conn]bool), + logger: logger, + } +} + +func (r *TcpReader) Listen() { + var err error + r.listener, err = net.Listen("tcp", fmt.Sprintf(":%d", r.port)) + if err != nil { + r.logger.Fatal("Server error: ", err) + } + + r.logger.Info("TCP server is listening on port %d\n", r.port) + + go func() { + for { + conn, err := r.listener.Accept() + if err != nil { + r.logger.Error("Error accepting connection: ", err) + continue + } + r.lock.Lock() + r.sockets[conn] = true + r.lock.Unlock() + + go r.handleConnection(conn) + } + }() +} + +func (r *TcpReader) handleConnection(conn net.Conn) { + address := conn.RemoteAddr().String() + r.logger.Debug("New TCP client connected from '%s'\n", address) + + scanner := bufio.NewScanner(conn) + for scanner.Scan() { + msg := scanner.Text() + // TODO: Process incoming message here + // For example, emit to a channel or call a callback function + fmt.Printf("Received message from '%s': %s\n", address, msg) + // + } + + if err := scanner.Err(); err != nil { + r.logger.Error("Error reading from '%s': %s\n", address, err) + } + + r.closeSocket(conn) +} + +func (r *TcpReader) closeSocket(conn net.Conn) { + conn.Close() + r.lock.Lock() + delete(r.sockets, conn) + r.lock.Unlock() +} + +func (r *TcpReader) Close() { + r.listener.Close() + for conn := range r.sockets { + r.closeSocket(conn) + } +} diff --git a/transit/tcp/tcp-writer.go b/transit/tcp_gpt/tcp-writer.go similarity index 100% rename from transit/tcp/tcp-writer.go rename to transit/tcp_gpt/tcp-writer.go diff --git a/transit/tcp/tcp.go b/transit/tcp_gpt/tcp.go similarity index 100% rename from transit/tcp/tcp.go rename to transit/tcp_gpt/tcp.go diff --git a/transit/tcp/udp.go b/transit/tcp_gpt/udp.go similarity index 75% rename from transit/tcp/udp.go rename to transit/tcp_gpt/udp.go index 10b2317..fabf58f 100644 --- a/transit/tcp/udp.go +++ b/transit/tcp_gpt/udp.go @@ -44,8 +44,10 @@ func (u *UdpServer) Bind() error { } u.conn = conn + //fiund our the internface (from interfaces, err := net.Interfaces() ) that corresponds to the addr ip address. + if u.opts.Multicast != "" { - err := u.joinMulticastGroup() + err := u.joinMulticastGroup()//sewndxnthe interface used bny the ser here (where is listening the UDP) if err != nil { u.logger.Println("Error joining multicast group:", err) // Handle non-fatal error, as we can continue in broadcast mode @@ -68,8 +70,9 @@ func (u *UdpServer) handleIncomingMessages() { continue } message := string(buffer[:n]) - u.logger.Printf("Received message from %s: %s", addr.String(), message) + u.logger.Debug("Received message from %s: %s", addr.String(), message) // Process message here + } } @@ -109,7 +112,29 @@ func (u *UdpServer) joinMulticastGroup() error { if multicastAddr == nil { return fmt.Errorf("invalid multicast address: %s", u.opts.Multicast) } - iface, err := net.InterfaceByName("eth0") // Specify the appropriate interface, or iterate over all if needed + + interfaces, err := net.Interfaces() + if err != nil { + return fmt.Errorf("failed to get interfaces: %v", err) + } + + p := ipv4.NewPacketConn(u.conn) + + for _, iface := range interfaces { + if err := p.JoinGroup(&iface, &net.UDPAddr{IP: multicastAddr}); err != nil { + u.logger.Printf("failed to join multicast group on interface %s: %v", iface.Name, err) + continue + } + if err := p.SetMulticastLoopback(true); err != nil { + u.logger.Printf("failed to set multicast loopback on interface %s: %v", iface.Name, err) + continue + } + } + + return nil +} + + iface, err := net.InterfaceByName("eth0") if err != nil { return fmt.Errorf("failed to get interface: %v", err) } From 1235a767d1adfd0081dd5dc97f059900cbc93bb8 Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Thu, 28 Mar 2024 18:47:04 +1300 Subject: [PATCH 03/20] tcp writter --- transit/tcp/gossip.go | 4 + transit/tcp/tcp-reader.go | 4 +- transit/tcp/tcp-transporter.go | 2 + transit/tcp/tcp-writer.go | 130 +++++++++++++++++++++++++++++++++ transit/tcp_gpt/tcp-reader.go | 85 --------------------- transit/tcp_gpt/tcp-writer.go | 104 -------------------------- 6 files changed, 138 insertions(+), 191 deletions(-) create mode 100644 transit/tcp/tcp-writer.go delete mode 100644 transit/tcp_gpt/tcp-reader.go delete mode 100644 transit/tcp_gpt/tcp-writer.go diff --git a/transit/tcp/gossip.go b/transit/tcp/gossip.go index 1915af9..a2aeb33 100644 --- a/transit/tcp/gossip.go +++ b/transit/tcp/gossip.go @@ -13,3 +13,7 @@ func (g *Gossip) processRequest(msgBytes *[]byte) { // processResponse func (g *Gossip) processResponse(msgBytes *[]byte) { } + +func isGossipMessage(msgType byte) bool { + return msgType == PACKET_GOSSIP_REQ || msgType == PACKET_GOSSIP_RES || msgType == PACKET_GOSSIP_HELLO +} diff --git a/transit/tcp/tcp-reader.go b/transit/tcp/tcp-reader.go index 88db887..681c98e 100644 --- a/transit/tcp/tcp-reader.go +++ b/transit/tcp/tcp-reader.go @@ -109,13 +109,13 @@ func (r *TcpReader) readMessage(conn net.Conn) (msgType int, msg []byte, err err // If the buffer is larger than the max packet size, return an error if r.maxPacketSize > 0 && len(buf) > r.maxPacketSize { - return 0, nil, fmt.Errorf("Incoming packet is larger than the 'maxPacketSize' limit (%d > %d)!", len(buf), r.maxPacketSize) + return 0, nil, fmt.Errorf("incoming packet is larger than the 'maxPacketSize' limit (%d > %d)", len(buf), r.maxPacketSize) } // Check the CRC crc := buf[1] ^ buf[2] ^ buf[3] ^ buf[4] ^ buf[5] if crc != buf[0] { - return 0, nil, fmt.Errorf("Invalid packet CRC! %d", crc) + return 0, nil, fmt.Errorf("invalid packet CRC %d", crc) } length := int(binary.BigEndian.Uint32(buf[1:])) diff --git a/transit/tcp/tcp-transporter.go b/transit/tcp/tcp-transporter.go index 2f29e43..f085fde 100644 --- a/transit/tcp/tcp-transporter.go +++ b/transit/tcp/tcp-transporter.go @@ -11,6 +11,7 @@ import ( type TCPTransporter struct { options TCPOptions tcpReader *TcpReader + tcpWriter *TcpWriter gossip *Gossip validateMsg transit.ValidateMsgFunc @@ -149,6 +150,7 @@ func (transporter *TCPTransporter) incomingMessage(msgType int, msgBytes *[]byte func (transporter *TCPTransporter) startTcpServer() { transporter.tcpReader = NewTcpReader(transporter.options.Port, transporter.onTcpMessage, transporter.options.Logger) + transporter.tcpWriter = NewTcpWriter(transporter.options.MaxConnections, transporter.options.Logger) } func (transporter *TCPTransporter) startUDPServer() { diff --git a/transit/tcp/tcp-writer.go b/transit/tcp/tcp-writer.go new file mode 100644 index 0000000..0fef29a --- /dev/null +++ b/transit/tcp/tcp-writer.go @@ -0,0 +1,130 @@ +package tcp + +import ( + "encoding/binary" + "errors" + "fmt" + "net" + "sync" + "time" + + "sort" + + log "github.com/sirupsen/logrus" +) + +const HEADER_SIZE = 6 + +type TCPConnEntry struct { + conn *net.TCPConn + lastUsed time.Time +} + +type TcpWriter struct { + sockets map[string]*TCPConnEntry + maxConnections int + logger *log.Entry + lock sync.Mutex +} + +func NewTcpWriter(maxConnections int, logger *log.Entry) *TcpWriter { + return &TcpWriter{ + sockets: make(map[string]*TCPConnEntry), + maxConnections: maxConnections, + logger: logger, + } +} + +func (w *TcpWriter) Connect(nodeID, host string, port int) (*net.TCPConn, error) { + w.lock.Lock() + defer w.lock.Unlock() + + if socket, exists := w.sockets[nodeID]; exists && socket != nil { + return socket.conn, nil + } + + addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", host, port)) + if err != nil { + return nil, err + } + + conn, err := net.DialTCP("tcp", nil, addr) + if err != nil { + return nil, err + } + + conn.SetNoDelay(true) + conn.SetKeepAlive(true) + conn.SetKeepAlivePeriod(3 * time.Minute) + + w.sockets[nodeID] = &TCPConnEntry{conn: conn, lastUsed: time.Now()} + + if len(w.sockets) > w.maxConnections { + w.manageConnections() + } + return conn, nil +} + +func (w *TcpWriter) Send(nodeID string, msgType byte, msgBytes []byte) error { + socket, exists := w.sockets[nodeID] + if !exists || socket == nil { + return errors.New("connection does not exist") + } + header := make([]byte, HEADER_SIZE) + binary.BigEndian.PutUint32(header[1:], uint32(len(msgBytes)+len(header))) + header[5] = msgType + crc := header[1] ^ header[2] ^ header[3] ^ header[4] ^ header[5] + header[0] = crc + + payload := append(header, msgBytes...) + _, err := socket.conn.Write(payload) + + if !isGossipMessage(msgType) { + socket.lastUsed = time.Now() + w.sockets[nodeID] = socket + } + return err +} + +type kv struct { + Key string + Value *TCPConnEntry +} + +func (w *TcpWriter) manageConnections() { + // Simplified version: Close excess connections + w.lock.Lock() + defer w.lock.Unlock() + + if len(w.sockets) <= w.maxConnections { + return + } + + orderedList := make([]kv, 0, len(w.sockets)) + for k, v := range w.sockets { + orderedList = append(orderedList, kv{k, v}) + } + sort.Slice(orderedList, func(i, j int) bool { + return orderedList[i].Value.lastUsed.Before(orderedList[j].Value.lastUsed) + }) + + for _, kv := range orderedList { + kv.Value.conn.Close() + nodeID := kv.Key + delete(w.sockets, nodeID) + w.logger.Debugf("Closed connection to node %s", nodeID) + if len(w.sockets) <= w.maxConnections { + break + } + } +} + +func (w *TcpWriter) Close() { + w.lock.Lock() + defer w.lock.Unlock() + + for nodeID, socket := range w.sockets { + socket.conn.Close() + delete(w.sockets, nodeID) + } +} diff --git a/transit/tcp_gpt/tcp-reader.go b/transit/tcp_gpt/tcp-reader.go deleted file mode 100644 index 90941af..0000000 --- a/transit/tcp_gpt/tcp-reader.go +++ /dev/null @@ -1,85 +0,0 @@ -package tcp - -import ( - "bufio" - "fmt" - "net" - "sync" - - log "github.com/sirupsen/logrus" -) - -type TcpReader struct { - port int - listener net.Listener - sockets map[net.Conn]bool - logger *log.Entry - lock sync.Mutex -} - -func NewTcpReader(port int, logger *log.Entry) *TcpReader { - return &TcpReader{ - port: port, - sockets: make(map[net.Conn]bool), - logger: logger, - } -} - -func (r *TcpReader) Listen() { - var err error - r.listener, err = net.Listen("tcp", fmt.Sprintf(":%d", r.port)) - if err != nil { - r.logger.Fatal("Server error: ", err) - } - - r.logger.Info("TCP server is listening on port %d\n", r.port) - - go func() { - for { - conn, err := r.listener.Accept() - if err != nil { - r.logger.Error("Error accepting connection: ", err) - continue - } - r.lock.Lock() - r.sockets[conn] = true - r.lock.Unlock() - - go r.handleConnection(conn) - } - }() -} - -func (r *TcpReader) handleConnection(conn net.Conn) { - address := conn.RemoteAddr().String() - r.logger.Debug("New TCP client connected from '%s'\n", address) - - scanner := bufio.NewScanner(conn) - for scanner.Scan() { - msg := scanner.Text() - // TODO: Process incoming message here - // For example, emit to a channel or call a callback function - fmt.Printf("Received message from '%s': %s\n", address, msg) - // - } - - if err := scanner.Err(); err != nil { - r.logger.Error("Error reading from '%s': %s\n", address, err) - } - - r.closeSocket(conn) -} - -func (r *TcpReader) closeSocket(conn net.Conn) { - conn.Close() - r.lock.Lock() - delete(r.sockets, conn) - r.lock.Unlock() -} - -func (r *TcpReader) Close() { - r.listener.Close() - for conn := range r.sockets { - r.closeSocket(conn) - } -} diff --git a/transit/tcp_gpt/tcp-writer.go b/transit/tcp_gpt/tcp-writer.go deleted file mode 100644 index 7e564a3..0000000 --- a/transit/tcp_gpt/tcp-writer.go +++ /dev/null @@ -1,104 +0,0 @@ -package tcp - -import ( - "encoding/binary" - "errors" - "fmt" - "net" - "sync" - "time" - - log "github.com/sirupsen/logrus" -) - -type TcpWriter struct { - sockets map[string]*net.TCPConn - opts WriterOptions - logger *log.Entry - lock sync.Mutex -} - -type WriterOptions struct { - MaxConnections int -} - -func NewTcpWriter(opts WriterOptions, logger *log.Entry) *TcpWriter { - return &TcpWriter{ - sockets: make(map[string]*net.TCPConn), - opts: opts, - logger: logger, - } -} - -func (w *TcpWriter) Connect(nodeID, host string, port int) (*net.TCPConn, error) { - w.lock.Lock() - defer w.lock.Unlock() - - if socket, exists := w.sockets[nodeID]; exists && socket != nil { - return socket, nil - } - - addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", host, port)) - if err != nil { - return nil, err - } - - conn, err := net.DialTCP("tcp", nil, addr) - if err != nil { - return nil, err - } - - conn.SetNoDelay(true) - conn.SetKeepAlive(true) - conn.SetKeepAlivePeriod(3 * time.Minute) - - w.sockets[nodeID] = conn - return conn, nil -} - -func (w *TcpWriter) Send(nodeID string, packetType byte, data []byte) error { - socket, exists := w.sockets[nodeID] - if !exists || socket == nil { - return errors.New("connection does not exist") - } - - header := make([]byte, 6) - binary.BigEndian.PutUint32(header[1:], uint32(len(data)+len(header))) - header[5] = packetType - crc := header[1] ^ header[2] ^ header[3] ^ header[4] ^ header[5] - header[0] = crc - - payload := append(header, data...) - _, err := socket.Write(payload) - return err -} - -func (w *TcpWriter) manageConnections() { - // Simplified version: Close excess connections - w.lock.Lock() - defer w.lock.Unlock() - - if len(w.sockets) <= w.opts.MaxConnections { - return - } - - for nodeID, socket := range w.sockets { - socket.Close() - delete(w.sockets, nodeID) - w.logger.Debug("Closed connection to node %s", nodeID) - - if len(w.sockets) <= w.opts.MaxConnections { - break - } - } -} - -func (w *TcpWriter) Close() { - w.lock.Lock() - defer w.lock.Unlock() - - for nodeID, socket := range w.sockets { - socket.Close() - delete(w.sockets, nodeID) - } -} From 99e7a779f3561fff502fb69186792031a86488be Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Thu, 28 Mar 2024 21:01:48 +1300 Subject: [PATCH 04/20] WIP udp --- moleculer.go | 6 + registry/node.go | 20 +++ registry/registry.go | 7 +- transit/nats/nats.go | 2 +- transit/nats/stan.go | 2 +- transit/tcp/tcp-transporter.go | 83 ++++++++++- transit/tcp/udp.go | 258 +++++++++++++++++++++++++++++++++ transit/tcp_gpt/parser.go | 71 --------- transit/tcp_gpt/tcp.go | 79 ---------- transit/tcp_gpt/udp.go | 149 ------------------- transit/transit.go | 4 +- 11 files changed, 369 insertions(+), 312 deletions(-) create mode 100644 transit/tcp/udp.go delete mode 100644 transit/tcp_gpt/parser.go delete mode 100644 transit/tcp_gpt/tcp.go delete mode 100644 transit/tcp_gpt/udp.go diff --git a/moleculer.go b/moleculer.go index bb9d44f..569010e 100644 --- a/moleculer.go +++ b/moleculer.go @@ -223,6 +223,7 @@ type Node interface { GetID() string ExportAsMap() map[string]interface{} IsAvailable() bool + GetIpList() []string Available() Unavailable() IsExpired(timeout time.Duration) bool @@ -250,6 +251,11 @@ type Context interface { Meta() Payload } +type Registry interface { + GetNodeByID(nodeID string) Node + AddOfflineNode(nodeID, address string, port int) Node +} + type BrokerContext interface { Call(actionName string, params interface{}, opts ...Options) chan Payload Emit(eventName string, params interface{}, groups ...string) diff --git a/registry/node.go b/registry/node.go index 33fc130..b7a12af 100644 --- a/registry/node.go +++ b/registry/node.go @@ -19,6 +19,8 @@ type Node struct { sequence int64 ipList []string hostname string + udpAddress string + port int client map[string]interface{} services []map[string]interface{} isAvailable bool @@ -80,6 +82,10 @@ func CreateNode(id string, local bool, logger *log.Entry) moleculer.Node { return result } +func (node *Node) GetIpList() []string { + return node.ipList +} + func (node *Node) Update(id string, info map[string]interface{}) (bool, []map[string]interface{}) { if id != node.id { panic(fmt.Errorf("Node.Update() - the id received : %s does not match this node.id : %s", id, node.id)) @@ -95,6 +101,19 @@ func (node *Node) Update(id string, info map[string]interface{}) (bool, []map[st node.ipList = interfaceToString(info["ipList"].([]interface{})) node.hostname = info["hostname"].(string) + + if port, ok := info["port"]; ok { + node.port = port.(int) + } + + if ipList, ok := info["ipList"]; ok { + node.ipList = ipList.([]string) + } + + if udpAddress, ok := info["udpAddress"]; ok { + node.udpAddress = udpAddress.(string) + } + node.client = info["client"].(map[string]interface{}) services, removedServices := FilterServices(node.services, info) @@ -174,6 +193,7 @@ func (node *Node) ExportAsMap() map[string]interface{} { resultMap["services"] = node.services // node.removeInternalServices(node.services) resultMap["ipList"] = node.ipList resultMap["hostname"] = node.hostname + resultMap["port"] = node.port resultMap["client"] = node.client resultMap["seq"] = node.sequence resultMap["cpu"] = node.cpu diff --git a/registry/registry.go b/registry/registry.go index d002de3..f9e14bc 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -147,11 +147,16 @@ func (registry *ServiceRegistry) LocalServices() []*service.Service { return []*service.Service{createNodeService(registry)} } +func (registry *ServiceRegistry) GetNodeByID(nodeID string) moleculer.Node { + node, _ := registry.nodes.findNode(nodeID) + return node +} + // Start : start the registry background processes. func (registry *ServiceRegistry) Start() { registry.logger.Debug("Registry Start() ") registry.stopping = false - err := <-registry.transit.Connect() + err := <-registry.transit.Connect(registry) if err != nil { panic(errors.New(fmt.Sprint("Could not connect to the transit. err: ", err))) } diff --git a/transit/nats/nats.go b/transit/nats/nats.go index 64facac..6f4e878 100644 --- a/transit/nats/nats.go +++ b/transit/nats/nats.go @@ -57,7 +57,7 @@ func CreateNatsTransporter(options NATSOptions) transit.Transport { } } -func (t *NatsTransporter) Connect() chan error { +func (t *NatsTransporter) Connect(registry moleculer.Registry) chan error { endChan := make(chan error) go func() { t.logger.Debug("NATS Connect() - url: ", t.opts.Url, " Name: ", t.opts.Name) diff --git a/transit/nats/stan.go b/transit/nats/stan.go index 6fb1656..9474146 100644 --- a/transit/nats/stan.go +++ b/transit/nats/stan.go @@ -46,7 +46,7 @@ func CreateStanTransporter(options StanOptions) StanTransporter { return transport } -func (transporter *StanTransporter) Connect() chan error { +func (transporter *StanTransporter) Connect(registry moleculer.Registry) chan error { endChan := make(chan error) go func() { transporter.logger.Debug("STAN Connect() - url: ", transporter.url, " clusterID: ", transporter.clusterID, " clientID: ", transporter.clientID) diff --git a/transit/tcp/tcp-transporter.go b/transit/tcp/tcp-transporter.go index f085fde..51bfbca 100644 --- a/transit/tcp/tcp-transporter.go +++ b/transit/tcp/tcp-transporter.go @@ -1,6 +1,8 @@ package tcp import ( + "time" + "github.com/moleculer-go/moleculer" "github.com/moleculer-go/moleculer/serializer" "github.com/moleculer-go/moleculer/transit" @@ -12,7 +14,11 @@ type TCPTransporter struct { options TCPOptions tcpReader *TcpReader tcpWriter *TcpWriter + udpServer *UdpServer gossip *Gossip + registry moleculer.Registry + + logger *log.Entry validateMsg transit.ValidateMsgFunc serializer serializer.Serializer @@ -31,7 +37,9 @@ type TCPOptions struct { // UDP bind address (if null, bind on all interfaces) UdpBindAddress string // UDP sending period (seconds) - UdpPeriod int + UdpPeriod time.Duration + + UdpMaxDiscovery int // Multicast address. UdpMulticast string @@ -39,8 +47,8 @@ type TCPOptions struct { UdpMulticastTTL int // Send broadcast (Boolean, String, Array) - UdpBroadcast bool - + UdpBroadcast []string + UdpBroadcastAddrs []string // TCP server port. Null or 0 means random port Port int // Static remote nodes address list (when UDP discovery is not available) @@ -63,12 +71,13 @@ type TCPOptions struct { } func CreateTCPTransporter(options TCPOptions) TCPTransporter { - transport := TCPTransporter{options: options} + transport := TCPTransporter{options: options, logger: options.Logger} return transport } -func (transporter *TCPTransporter) Connect() chan error { - transporter.options.Logger.Info("TCP Transported Connect()") +func (transporter *TCPTransporter) Connect(registry moleculer.Registry) chan error { + transporter.registry = registry + transporter.logger.Info("TCP Transported Connect()") endChan := make(chan error) go func() { transporter.startTcpServer() @@ -149,12 +158,70 @@ func (transporter *TCPTransporter) incomingMessage(msgType int, msgBytes *[]byte } func (transporter *TCPTransporter) startTcpServer() { - transporter.tcpReader = NewTcpReader(transporter.options.Port, transporter.onTcpMessage, transporter.options.Logger) - transporter.tcpWriter = NewTcpWriter(transporter.options.MaxConnections, transporter.options.Logger) + transporter.tcpReader = NewTcpReader(transporter.options.Port, transporter.onTcpMessage, transporter.logger.WithFields(log.Fields{ + "TCPTransporter": "TCPReader", + })) + transporter.tcpWriter = NewTcpWriter(transporter.options.MaxConnections, transporter.logger.WithFields(log.Fields{ + "TCPTransporter": "TCPWriter", + })) } func (transporter *TCPTransporter) startUDPServer() { + transporter.udpServer = NewUdpServer(UdpServerOptions{ + Port: transporter.options.UdpPort, + BindAddress: transporter.options.UdpBindAddress, + Multicast: transporter.options.UdpMulticast, + MulticastTTL: transporter.options.UdpMulticastTTL, + BroadcastAddrs: transporter.options.UdpBroadcast, + DiscoverPeriod: transporter.options.UdpPeriod, + MaxDiscovery: transporter.options.UdpMaxDiscovery, + Discovery: transporter.options.UdpDiscovery, + }, transporter.logger.WithFields(log.Fields{ + "TCPTransporter": "UdpServer", + })) + + err := transporter.udpServer.Start() + if err != nil { + transporter.logger.Error("Error starting UDP server:", err) + } + +} + +func (transporter *TCPTransporter) onUdpMessage(nodeID, address string, port int) { + if nodeID != "" && nodeID != transporter.options.NodeId { + transporter.logger.Debug(`UDP discovery received from ${address} on ${nodeID}.`) + + node := transporter.registry.GetNodeByID(nodeID) + if node == nil { + // Unknown node. Register as offline node + node = transporter.registry.AddOfflineNode(nodeID, address, port) + } else if !node.IsAvailable() { + ipList := node.GetIpList() + found := false + for i, ip := range ipList { + if ip == address { + // Move the address to the front of the list + ipList = append([]string{address}, append(ipList[:i], ipList[i+1:]...)...) + found = true + break + } + } + + if !found { + // If the address is not in the list, add it to the front + ipList = append([]string{address}, ipList...) + } + node.Update(nodeID, map[string]interface{}{ + "hostname": address, + "port": port, + "ipList": ipList, + }) + } + node.Update(nodeID, map[string]interface{}{ + "udpAddress": address, + }) + } } func (transporter *TCPTransporter) startGossip() { diff --git a/transit/tcp/udp.go b/transit/tcp/udp.go new file mode 100644 index 0000000..b6b2cb4 --- /dev/null +++ b/transit/tcp/udp.go @@ -0,0 +1,258 @@ +package tcp + +import ( + "fmt" + "net" + "strconv" + "strings" + "time" + + "golang.org/x/net/ipv4" + + log "github.com/sirupsen/logrus" +) + +type UdpServer struct { + conn *net.UDPConn + opts UdpServerOptions + discoveryCounter int + logger *log.Entry + discoverTimer *time.Ticker + servers []*net.UDPConn +} + +type UdpServerOptions struct { + Port int + Multicast string + MulticastTTL int + BindAddress string + BroadcastAddrs []string + DiscoverPeriod time.Duration + MaxDiscovery int + Discovery bool +} + +func NewUdpServer(opts UdpServerOptions, logger *log.Entry) *UdpServer { + return &UdpServer{ + opts: opts, + logger: logger, + } +} + +func (u *UdpServer) startServer(ip string, port int, multicast string, multicastTTL int) error { + udpAddr, err := net.ResolveUDPAddr("udp4", fmt.Sprintf("%s:%d", ip, port)) + if err != nil { + u.logger.Warnf("Unable to resolve UDP address: %s\n", err) + return err + } + + udpConn, err := net.ListenUDP("udp4", udpAddr) + if err != nil { + u.logger.Warnf("Unable to listen on UDP address: %s\n", err) + return err + } + + if multicast != "" { + u.logger.Infof("UDP Multicast Server is listening on %s:%d. Membership: %s \n", ip, port, multicast) + err := u.joinMulticastGroup(multicast, udpConn, multicastTTL, ip, port) + if err != nil { + u.logger.Error("Error joining multicast group:", err) + return err + } + + } else { + u.logger.Infof("UDP Broadcast Server is listening on %s:%d\n", ip, port) + } + + // Start a goroutine to handle incoming messages + go func() { + buf := make([]byte, 1024) + for { + n, addr, err := udpConn.ReadFromUDP(buf) + if err != nil { + u.logger.Warnf("Error reading from UDP: %s\n", err) + break + } + + // Handle the message + u.onMessage(buf[:n], addr) + } + }() + + // Add the connection to your list of servers + u.servers = append(u.servers, udpConn) + + return nil +} + +func (u *UdpServer) onMessage(buf []byte, addr *net.UDPAddr) { + msg := string(buf) + u.logger.Debugf("UDP message received from %s: %s\n", addr.String(), msg) + + parts := strings.Split(msg, "|") + if len(parts) != 3 { + u.logger.Debugf("Malformed UDP packet received: %s\n", msg) + return + } + + if parts[0] == u.namespace { + port, err := strconv.Atoi(parts[2]) + if err != nil { + u.logger.Debugf("UDP packet process error: %s\n", err) + return + } + + u.emit("message", parts[1], addr.IP.String(), port) + } +} + +func (u *UdpServer) joinMulticastGroup(multicast string, udpConn *net.UDPConn, multicastTTL int, ip string, port int) error { + groupAddr, err := net.ResolveUDPAddr("udp4", multicast) + if err != nil { + u.logger.Warnf("Unable to resolve multicast address: %s\n", err) + return err + } + + p := ipv4.NewPacketConn(udpConn) + + interfaces, err := net.Interfaces() + if err != nil { + u.logger.Warnf("Unable to get network interfaces: %s\n", err) + return err + } + + for _, iface := range interfaces { + if err := p.JoinGroup(&iface, groupAddr); err != nil { + u.logger.Warnf("Unable to join multicast group on interface %s: %s\n", iface.Name, err) + } + } + + if err := p.SetMulticastTTL(multicastTTL); err != nil { + u.logger.Warnf("Unable to set multicast TTL: %s\n", err) + return err + } + + u.logger.Infof("UDP Multicast Server is listening on %s:%d. Membership: %s\n", ip, port, multicast) + return nil +} + +func (u *UdpServer) getAllIPs() []string { + ips := []string{} + interfaces, err := net.Interfaces() + if err != nil { + u.logger.Error("Error getting interfaces:", err) + return ips + } + + for _, i := range interfaces { + addrs, err := i.Addrs() + if err != nil { + u.logger.Error("Error getting addresses for interface:", err) + continue + } + + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + + if ip == nil { + continue + } + ips = append(ips, ip.String()) + u.logger.Debug("Interface: %v, IP Address: %v", i.Name, ip.String()) + } + } + return ips +} + +func (u *UdpServer) Start() error { + if u.opts.Multicast != "" { + if u.opts.BindAddress != "" { + // Bind only one interface + return u.startServer(u.opts.BindAddress, u.opts.Port, u.opts.Multicast, u.opts.MulticastTTL) + } + //list all interfaces and the ip addresses of each interface + ips := u.getAllIPs() + for _, ip := range ips { + err := u.startServer(ip, u.opts.Port, u.opts.Multicast, u.opts.MulticastTTL) + if err != nil { + u.logger.Error("Error starting server on IP:", ip, err) + } + } + } else if len(u.opts.BroadcastAddrs) > 0 { + return u.startServer(u.opts.BindAddress, u.opts.Port, "", 0) + } + + go u.firstDiscoveryMessage() + + go u.handleIncomingMessages() + + u.startDiscovering() + + return nil +} + +func (u *UdpServer) firstDiscoveryMessage() { + //wait for 1 second before sending the first discovery message + time.Sleep(time.Second) + u.broadcastDiscoveryMessage() +} + +func (u *UdpServer) handleIncomingMessages() { + buffer := make([]byte, 2048) + for { + n, addr, err := u.conn.ReadFromUDP(buffer) + if err != nil { + u.logger.Println("Error reading from UDP:", err) + continue + } + message := string(buffer[:n]) + u.logger.Debug("Received message from %s: %s", addr.String(), message) + // Process message here + + } +} + +func (u *UdpServer) startDiscovering() { + if u.opts.Discovery == false { + u.logger.Info("UDP Discovery is disabled.") + return + } + u.discoverTimer = time.NewTicker(u.opts.DiscoverPeriod) + go func() { + for range u.discoverTimer.C { + u.broadcastDiscoveryMessage() + if u.opts.MaxDiscovery > 0 && u.discoveryCounter >= u.opts.MaxDiscovery { + u.StopDiscovering() + } + } + }() +} + +func (u *UdpServer) broadcastDiscoveryMessage() { + message := []byte("discovery message") // Customize your message + destAddr := &net.UDPAddr{IP: net.IPv4bcast, Port: u.opts.Port} + if _, err := u.conn.WriteToUDP(message, destAddr); err != nil { + u.logger.Println("Error broadcasting discovery message:", err) + } +} + +func (u *UdpServer) StopDiscovering() { + if u.discoverTimer != nil { + u.discoverTimer.Stop() + u.discoverTimer = nil + u.logger.Info("Discovery limit reached, stopping UDP discovery") + } +} + +func (u *UdpServer) Close() { + u.StopDiscovering() + if u.conn != nil { + u.conn.Close() + } +} diff --git a/transit/tcp_gpt/parser.go b/transit/tcp_gpt/parser.go deleted file mode 100644 index 7ecc8fd..0000000 --- a/transit/tcp_gpt/parser.go +++ /dev/null @@ -1,71 +0,0 @@ -package tcp - -import ( - "encoding/binary" - "errors" - "io" - "net" - - log "github.com/sirupsen/logrus" -) - -type Parser struct { - conn *net.TCPConn - maxPacketSize int - logger *log.Entry -} - -func NewParser(conn *net.TCPConn, maxPacketSize int, logger *log.Entry) *Parser { - return &Parser{ - conn: conn, - maxPacketSize: maxPacketSize, - logger: logger, - } -} - -const HeaderSize = 6 - -func (p *Parser) StartParsing(handler func(packetType byte, data []byte)) error { - buf := make([]byte, 0, 4096) // Initial buffer size, adjust based on needs - tmp := make([]byte, 1024) // Temporary buffer for reading from connection - - for { - n, err := p.conn.Read(tmp) - if err != nil { - if err != io.EOF { - return err // Handle error (excluding EOF) - } - break - } - - buf = append(buf, tmp[:n]...) - - for { - if len(buf) < HeaderSize { - break // Wait for more data - } - - if p.maxPacketSize > 0 && len(buf) > p.maxPacketSize { - return errors.New("incoming packet is larger than the 'maxPacketSize' limit") - } - - length := int(binary.BigEndian.Uint32(buf[1:5])) - if len(buf) < length { - break // Wait for complete packet - } - - crc := buf[1] ^ buf[2] ^ buf[3] ^ buf[4] ^ buf[5] - if crc != buf[0] { - return errors.New("invalid packet CRC") - } - - packetType := buf[5] - data := buf[HeaderSize:length] - handler(packetType, data) - - buf = buf[length:] // Move to next packet - } - } - - return nil -} diff --git a/transit/tcp_gpt/tcp.go b/transit/tcp_gpt/tcp.go deleted file mode 100644 index 95ef64d..0000000 --- a/transit/tcp_gpt/tcp.go +++ /dev/null @@ -1,79 +0,0 @@ -package tcp - -import ( - "time" - - log "github.com/sirupsen/logrus" -) - -type TcpTransporter struct { - opts TransporterOptions - reader *TcpReader // Placeholder for actual implementation - writer *TcpWriter // Placeholder for actual implementation - udpServer *UdpServer // Placeholder for actual implementation - gossipTimer *time.Ticker - logger *log.Entry -} - -type TransporterOptions struct { - UdpDiscovery bool - UdpPort int - UdpBindAddress string - UdpPeriod time.Duration - UdpReuseAddr bool - UdpMaxDiscovery int - UdpMulticast string - UdpMulticastTTL int - UdpBroadcast bool - Port int - Urls []string - UseHostname bool - GossipPeriod time.Duration - MaxConnections int - MaxPacketSize int - Logger *log.Entry -} - -func NewTcpTransporter(options TransporterOptions) *TcpTransporter { - return &TcpTransporter{ - opts: options, - logger: options.Logger, - } -} - -func (t *TcpTransporter) Connect() error { - // Example: Starting TCP and UDP server in Go - if err := t.startTcpServer(); err != nil { - return err - } - if t.opts.UdpDiscovery { - if err := t.startUdpServer(); err != nil { - return err - } - } - t.startTimers() - t.logger.Info("TCP Transporter started.") - // Additional setup and connection logic goes here - return nil -} - -// Placeholder for the actual TCP and UDP start methods -func (t *TcpTransporter) startTcpServer() error { - // TCP server setup and start logic - return nil -} - -func (t *TcpTransporter) startUdpServer() error { - // UDP server setup and start logic - return nil -} - -func (t *TcpTransporter) startTimers() { - // Example: Starting a ticker for gossip protocol - t.gossipTimer = time.NewTicker(t.opts.GossipPeriod * time.Second) - go func() { - for range t.gossipTimer.C { - // Handle gossip timer tick - } - }() -} diff --git a/transit/tcp_gpt/udp.go b/transit/tcp_gpt/udp.go deleted file mode 100644 index fabf58f..0000000 --- a/transit/tcp_gpt/udp.go +++ /dev/null @@ -1,149 +0,0 @@ -package tcp - -import ( - "fmt" - "net" - "time" - - "golang.org/x/net/ipv4" - - log "github.com/sirupsen/logrus" -) - -type UdpServer struct { - conn *net.UDPConn - opts UdpServerOptions - logger *log.Entry - discoverTimer *time.Ticker -} - -type UdpServerOptions struct { - Port int - Multicast string - MulticastTTL int - Broadcast bool - BroadcastAddrs []string // Optional, specify if not broadcasting to default subnet broadcast addresses - DiscoverPeriod time.Duration -} - -func NewUdpServer(opts UdpServerOptions, logger *log.Entry) *UdpServer { - return &UdpServer{ - opts: opts, - logger: logger, - } -} - -func (u *UdpServer) Bind() error { - addr := net.UDPAddr{ - Port: u.opts.Port, - IP: net.ParseIP("0.0.0.0"), - } - conn, err := net.ListenUDP("udp", &addr) - if err != nil { - return err - } - u.conn = conn - - //fiund our the internface (from interfaces, err := net.Interfaces() ) that corresponds to the addr ip address. - - if u.opts.Multicast != "" { - err := u.joinMulticastGroup()//sewndxnthe interface used bny the ser here (where is listening the UDP) - if err != nil { - u.logger.Println("Error joining multicast group:", err) - // Handle non-fatal error, as we can continue in broadcast mode - } - } - - go u.handleIncomingMessages() - - u.startDiscovering() - - return nil -} - -func (u *UdpServer) handleIncomingMessages() { - buffer := make([]byte, 2048) - for { - n, addr, err := u.conn.ReadFromUDP(buffer) - if err != nil { - u.logger.Println("Error reading from UDP:", err) - continue - } - message := string(buffer[:n]) - u.logger.Debug("Received message from %s: %s", addr.String(), message) - // Process message here - - } -} - -func (u *UdpServer) startDiscovering() { - u.discoverTimer = time.NewTicker(u.opts.DiscoverPeriod) - go func() { - for range u.discoverTimer.C { - u.broadcastDiscoveryMessage() - } - }() -} - -func (u *UdpServer) broadcastDiscoveryMessage() { - message := []byte("discovery message") // Customize your message - destAddr := &net.UDPAddr{IP: net.IPv4bcast, Port: u.opts.Port} - if _, err := u.conn.WriteToUDP(message, destAddr); err != nil { - u.logger.Println("Error broadcasting discovery message:", err) - } -} - -func (u *UdpServer) StopDiscovering() { - if u.discoverTimer != nil { - u.discoverTimer.Stop() - u.discoverTimer = nil - } -} - -func (u *UdpServer) Close() { - u.StopDiscovering() - if u.conn != nil { - u.conn.Close() - } -} - -func (u *UdpServer) joinMulticastGroup() error { - multicastAddr := net.ParseIP(u.opts.Multicast) - if multicastAddr == nil { - return fmt.Errorf("invalid multicast address: %s", u.opts.Multicast) - } - - interfaces, err := net.Interfaces() - if err != nil { - return fmt.Errorf("failed to get interfaces: %v", err) - } - - p := ipv4.NewPacketConn(u.conn) - - for _, iface := range interfaces { - if err := p.JoinGroup(&iface, &net.UDPAddr{IP: multicastAddr}); err != nil { - u.logger.Printf("failed to join multicast group on interface %s: %v", iface.Name, err) - continue - } - if err := p.SetMulticastLoopback(true); err != nil { - u.logger.Printf("failed to set multicast loopback on interface %s: %v", iface.Name, err) - continue - } - } - - return nil -} - - iface, err := net.InterfaceByName("eth0") - if err != nil { - return fmt.Errorf("failed to get interface: %v", err) - } - p := ipv4.NewPacketConn(u.conn) - if err := p.JoinGroup(iface, &net.UDPAddr{IP: multicastAddr}); err != nil { - return fmt.Errorf("failed to join multicast group: %v", err) - } - if err := p.SetMulticastLoopback(true); err != nil { - return fmt.Errorf("failed to set multicast loopback: %v", err) - } - return nil -} diff --git a/transit/transit.go b/transit/transit.go index 3f46547..4df1b07 100644 --- a/transit/transit.go +++ b/transit/transit.go @@ -12,7 +12,7 @@ type ValidateMsgFunc func(moleculer.Payload) bool type Transit interface { Emit(moleculer.BrokerContext) Request(moleculer.BrokerContext) chan moleculer.Payload - Connect() chan error + Connect(moleculer.Registry) chan error Disconnect() chan error DiscoverNode(nodeID string) @@ -22,7 +22,7 @@ type Transit interface { } type Transport interface { - Connect() chan error + Connect(registry moleculer.Registry) chan error Disconnect() chan error Subscribe(command, nodeID string, handler TransportHandler) Publish(command, nodeID string, message moleculer.Payload) From 6829f9882f2787c44a4da58d9e9631c86f554b26 Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Wed, 10 Apr 2024 12:37:13 +1200 Subject: [PATCH 05/20] handle udp message --- moleculer.go | 2 +- registry/node.go | 93 +++++++++++++++++++++------------- registry/registry.go | 12 +++++ transit/pubsub/pubsub.go | 4 +- transit/tcp/tcp-transporter.go | 37 +++++++------- 5 files changed, 93 insertions(+), 55 deletions(-) diff --git a/moleculer.go b/moleculer.go index 569010e..016384f 100644 --- a/moleculer.go +++ b/moleculer.go @@ -228,7 +228,7 @@ type Node interface { Unavailable() IsExpired(timeout time.Duration) bool Update(id string, info map[string]interface{}) (bool, []map[string]interface{}) - + UpdateInfo(id string, info map[string]interface{}) []map[string]interface{} IncreaseSequence() HeartBeat(heartbeat map[string]interface{}) Publish(service map[string]interface{}) diff --git a/registry/node.go b/registry/node.go index b7a12af..d6461ca 100644 --- a/registry/node.go +++ b/registry/node.go @@ -61,8 +61,6 @@ func discoverHostname() string { } func CreateNode(id string, local bool, logger *log.Entry) moleculer.Node { - ipList := discoverIpList() - hostname := discoverHostname() services := make([]map[string]interface{}, 0) node := Node{ id: id, @@ -71,13 +69,18 @@ func CreateNode(id string, local bool, logger *log.Entry) moleculer.Node { "version": version.Moleculer(), "langVersion": version.Go(), }, - ipList: ipList, - hostname: hostname, services: services, logger: logger, - isLocal: local, sequence: 1, } + if local { + node.isLocal = true + node.ipList = discoverIpList() + node.hostname = discoverHostname() + if node.hostname == "unknown" && len(node.ipList) > 0 { + node.hostname = node.ipList[0] + } + } var result moleculer.Node = &node return result } @@ -86,44 +89,64 @@ func (node *Node) GetIpList() []string { return node.ipList } -func (node *Node) Update(id string, info map[string]interface{}) (bool, []map[string]interface{}) { +func (node *Node) UpdateInfo(id string, info map[string]interface{}) []map[string]interface{} { if id != node.id { - panic(fmt.Errorf("Node.Update() - the id received : %s does not match this node.id : %s", id, node.id)) + node.logger.Error(fmt.Sprintf("Node.Update() - the id received : %s does not match this node.id : %s", id, node.id)) + return []map[string]interface{}{} } - node.logger.Debug("node.Update() - info:") + node.logger.Debug("node.UpdateInfo() - info:") node.logger.Debug(util.PrettyPrintMap(info)) - reconnected := !node.isAvailable - - node.isAvailable = true - node.lastHeartBeatTime = time.Now().Unix() - node.offlineSince = 0 - - node.ipList = interfaceToString(info["ipList"].([]interface{})) - node.hostname = info["hostname"].(string) - - if port, ok := info["port"]; ok { - node.port = port.(int) + if ipListArray, ok := info["ipList"].([]interface{}); ok { + node.ipList = interfaceToString(ipListArray) } - - if ipList, ok := info["ipList"]; ok { - node.ipList = ipList.([]string) + if ipList, ok := info["ipList"].([]string); ok { + node.ipList = ipList } - - if udpAddress, ok := info["udpAddress"]; ok { - node.udpAddress = udpAddress.(string) + if hostname, ok := info["hostname"].(string); ok { + node.hostname = hostname } + if local, ok := info["local"].(bool); ok { + node.isLocal = local + } + if port, ok := info["port"].(int); ok { + node.port = port + } + if udpAddress, ok := info["udpAddress"].(string); ok { + node.udpAddress = udpAddress + } + if client, ok := info["client"].(map[string]interface{}); ok { + node.client = client + } + if _, ok := info["seq"]; !ok { + node.sequence = int64Field(info, "seq", 0) + } + if _, ok := info["cpu"]; !ok { + node.cpu = int64Field(info, "cpu", 0) + } + if _, ok := info["cpuSeq"]; !ok { + node.cpuSequence = int64Field(info, "cpuSeq", 0) + } + if _, ok := info["services"].([]interface{}); ok { + services, removedServices := FilterServices(node.services, info) + node.logger.Debug("removedServices:", util.PrettyPrintMap(removedServices)) + node.services = services + return removedServices + } + return []map[string]interface{}{} +} - node.client = info["client"].(map[string]interface{}) - - services, removedServices := FilterServices(node.services, info) - node.logger.Debug("removedServices:", util.PrettyPrintMap(removedServices)) - - node.services = services - node.sequence = int64Field(info, "seq", 0) - node.cpu = int64Field(info, "cpu", 0) - node.cpuSequence = int64Field(info, "cpuSeq", 0) - +func (node *Node) Update(id string, info map[string]interface{}) (bool, []map[string]interface{}) { + if id != node.id { + node.logger.Error(fmt.Sprintf("Node.Update() - the id received : %s does not match this node.id : %s", id, node.id)) + return false, nil + } + node.logger.Debug("node.Update()") + removedServices := node.UpdateInfo(id, info) + reconnected := !node.isAvailable + node.isAvailable = true + node.lastHeartBeatTime = time.Now().Unix() + node.offlineSince = 0 return reconnected, removedServices } diff --git a/registry/registry.go b/registry/registry.go index f9e14bc..9944cce 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -113,6 +113,18 @@ func (registry *ServiceRegistry) LocalNode() moleculer.Node { return registry.localNode } +// Register a node as offline because we don't know all information about it +func (registry *ServiceRegistry) AddOfflineNode(nodeID string, address string, port int) moleculer.Node { + node := CreateNode(nodeID, false, registry.logger.WithField("Node", nodeID)) + node.UpdateInfo(nodeID, map[string]interface{}{ + "hostname": address, + "port": port, + "ipList": []string{address}, + }) + registry.nodes.Add(node) + return node +} + func (registry *ServiceRegistry) setupMessageHandlers() { messageHandler := map[string]messageHandlerFunc{ "HEARTBEAT": registry.filterMessages(registry.heartbeatMessageReceived), diff --git a/transit/pubsub/pubsub.go b/transit/pubsub/pubsub.go index c25fc92..bd46639 100644 --- a/transit/pubsub/pubsub.go +++ b/transit/pubsub/pubsub.go @@ -759,7 +759,7 @@ func (pubsub *PubSub) Disconnect() chan error { } // Connect : connect the transit with the transporter, subscribe to all events and start publishing its node info -func (pubsub *PubSub) Connect() chan error { +func (pubsub *PubSub) Connect(registry moleculer.Registry) chan error { endChan := make(chan error) if pubsub.isConnected { endChan <- nil @@ -768,7 +768,7 @@ func (pubsub *PubSub) Connect() chan error { pubsub.logger.Debug("PubSub - Connecting transport...") pubsub.transport = pubsub.createTransport() go func() { - err := <-pubsub.transport.Connect() + err := <-pubsub.transport.Connect(registry) if err == nil { pubsub.isConnected = true pubsub.logger.Debug("PubSub - Transport Connected!") diff --git a/transit/tcp/tcp-transporter.go b/transit/tcp/tcp-transporter.go index 51bfbca..e96c1da 100644 --- a/transit/tcp/tcp-transporter.go +++ b/transit/tcp/tcp-transporter.go @@ -188,6 +188,23 @@ func (transporter *TCPTransporter) startUDPServer() { } +func addIpToList(ipList []string, address string) []string { + found := false + for i, ip := range ipList { + if ip == address { + // Move the address to the front of the list + ipList = append([]string{address}, append(ipList[:i], ipList[i+1:]...)...) + found = true + break + } + } + if !found { + // If the address is not in the list, add it to the front + ipList = append([]string{address}, ipList...) + } + return ipList +} + func (transporter *TCPTransporter) onUdpMessage(nodeID, address string, port int) { if nodeID != "" && nodeID != transporter.options.NodeId { transporter.logger.Debug(`UDP discovery received from ${address} on ${nodeID}.`) @@ -197,28 +214,14 @@ func (transporter *TCPTransporter) onUdpMessage(nodeID, address string, port int // Unknown node. Register as offline node node = transporter.registry.AddOfflineNode(nodeID, address, port) } else if !node.IsAvailable() { - ipList := node.GetIpList() - found := false - for i, ip := range ipList { - if ip == address { - // Move the address to the front of the list - ipList = append([]string{address}, append(ipList[:i], ipList[i+1:]...)...) - found = true - break - } - } - - if !found { - // If the address is not in the list, add it to the front - ipList = append([]string{address}, ipList...) - } - node.Update(nodeID, map[string]interface{}{ + ipList := addIpToList(node.GetIpList(), address) + node.UpdateInfo(nodeID, map[string]interface{}{ "hostname": address, "port": port, "ipList": ipList, }) } - node.Update(nodeID, map[string]interface{}{ + node.UpdateInfo(nodeID, map[string]interface{}{ "udpAddress": address, }) } From 98c3882d22f20d0fdeeace600c442dfc67a7e2de Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Wed, 10 Apr 2024 12:39:44 +1200 Subject: [PATCH 06/20] chore --- transit/tcp/tcp-transporter.go | 1 - 1 file changed, 1 deletion(-) diff --git a/transit/tcp/tcp-transporter.go b/transit/tcp/tcp-transporter.go index e96c1da..d123627 100644 --- a/transit/tcp/tcp-transporter.go +++ b/transit/tcp/tcp-transporter.go @@ -208,7 +208,6 @@ func addIpToList(ipList []string, address string) []string { func (transporter *TCPTransporter) onUdpMessage(nodeID, address string, port int) { if nodeID != "" && nodeID != transporter.options.NodeId { transporter.logger.Debug(`UDP discovery received from ${address} on ${nodeID}.`) - node := transporter.registry.GetNodeByID(nodeID) if node == nil { // Unknown node. Register as offline node From 3187242413711e25225c015f1f0b3f954e4a526b Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Wed, 10 Apr 2024 14:20:52 +1200 Subject: [PATCH 07/20] udp server --- transit/pubsub/pubsub.go | 4 +- transit/tcp/tcp-transporter.go | 4 +- transit/tcp/udp.go | 204 ++++++++++++++++++++++++--------- 3 files changed, 152 insertions(+), 60 deletions(-) diff --git a/transit/pubsub/pubsub.go b/transit/pubsub/pubsub.go index bd46639..45ad532 100644 --- a/transit/pubsub/pubsub.go +++ b/transit/pubsub/pubsub.go @@ -238,7 +238,7 @@ func (pubsub *PubSub) createTCPTransporter() transit.Transport { // UDP port UdpPort: 4445, - // UDP bind address (if null, bind on all interfaces) + // UDP bind address (if empty + UdpMulticast is specified, bind on all interfaces) UdpBindAddress: "", // UDP sending period (seconds) UdpPeriod: 30, @@ -249,7 +249,7 @@ func (pubsub *PubSub) createTCPTransporter() transit.Transport { UdpMulticastTTL: 1, // Send broadcast (Boolean, String, Array) - UdpBroadcast: false, + UdpBroadcast: []string{}, // TCP server port. 0 means random port Port: 0, diff --git a/transit/tcp/tcp-transporter.go b/transit/tcp/tcp-transporter.go index d123627..3e3b4e0 100644 --- a/transit/tcp/tcp-transporter.go +++ b/transit/tcp/tcp-transporter.go @@ -167,7 +167,6 @@ func (transporter *TCPTransporter) startTcpServer() { } func (transporter *TCPTransporter) startUDPServer() { - transporter.udpServer = NewUdpServer(UdpServerOptions{ Port: transporter.options.UdpPort, BindAddress: transporter.options.UdpBindAddress, @@ -177,7 +176,8 @@ func (transporter *TCPTransporter) startUDPServer() { DiscoverPeriod: transporter.options.UdpPeriod, MaxDiscovery: transporter.options.UdpMaxDiscovery, Discovery: transporter.options.UdpDiscovery, - }, transporter.logger.WithFields(log.Fields{ + // Namespace: transporter.options.Namespace, TODO + }, transporter.onUdpMessage, transporter.logger.WithFields(log.Fields{ "TCPTransporter": "UdpServer", })) diff --git a/transit/tcp/udp.go b/transit/tcp/udp.go index b6b2cb4..abb34ff 100644 --- a/transit/tcp/udp.go +++ b/transit/tcp/udp.go @@ -12,13 +12,21 @@ import ( log "github.com/sirupsen/logrus" ) -type UdpServer struct { +type UdpServerEntry struct { conn *net.UDPConn + discoveryTargets []string +} + +type OnUdpMessage func(nodeID, ip string, port int) + +type UdpServer struct { + state string opts UdpServerOptions discoveryCounter int logger *log.Entry discoverTimer *time.Ticker - servers []*net.UDPConn + servers []*UdpServerEntry + onUdpMessage OnUdpMessage } type UdpServerOptions struct { @@ -30,16 +38,19 @@ type UdpServerOptions struct { DiscoverPeriod time.Duration MaxDiscovery int Discovery bool + Namespace string + NodeID string } -func NewUdpServer(opts UdpServerOptions, logger *log.Entry) *UdpServer { +func NewUdpServer(opts UdpServerOptions, onUdpMessage OnUdpMessage, logger *log.Entry) *UdpServer { return &UdpServer{ - opts: opts, - logger: logger, + opts: opts, + onUdpMessage: onUdpMessage, + logger: logger, } } -func (u *UdpServer) startServer(ip string, port int, multicast string, multicastTTL int) error { +func (u *UdpServer) startServer(ip string, port int, multicast string, multicastTTL int, discoveryTargets []string) error { udpAddr, err := net.ResolveUDPAddr("udp4", fmt.Sprintf("%s:%d", ip, port)) if err != nil { u.logger.Warnf("Unable to resolve UDP address: %s\n", err) @@ -64,47 +75,45 @@ func (u *UdpServer) startServer(ip string, port int, multicast string, multicast u.logger.Infof("UDP Broadcast Server is listening on %s:%d\n", ip, port) } - // Start a goroutine to handle incoming messages - go func() { - buf := make([]byte, 1024) - for { - n, addr, err := udpConn.ReadFromUDP(buf) - if err != nil { - u.logger.Warnf("Error reading from UDP: %s\n", err) - break - } - - // Handle the message - u.onMessage(buf[:n], addr) - } - }() - + // // Start a goroutine to handle incoming messages + // go func() { + // buf := make([]byte, 1024) + // for { + // n, addr, err := udpConn.ReadFromUDP(buf) + // if err != nil { + // u.logger.Warnf("Error reading from UDP: %s\n", err) + // break + // } + + // // Handle the message + // u.onMessage(buf[:n], addr) + // } + // }() // Add the connection to your list of servers - u.servers = append(u.servers, udpConn) - + u.servers = append(u.servers, &UdpServerEntry{udpConn, discoveryTargets}) return nil } -func (u *UdpServer) onMessage(buf []byte, addr *net.UDPAddr) { - msg := string(buf) - u.logger.Debugf("UDP message received from %s: %s\n", addr.String(), msg) +// func (u *UdpServer) onMessage(buf []byte, addr *net.UDPAddr) { +// msg := string(buf) +// u.logger.Debugf("UDP message received from %s: %s\n", addr.String(), msg) - parts := strings.Split(msg, "|") - if len(parts) != 3 { - u.logger.Debugf("Malformed UDP packet received: %s\n", msg) - return - } +// parts := strings.Split(msg, "|") +// if len(parts) != 3 { +// u.logger.Debugf("Malformed UDP packet received: %s\n", msg) +// return +// } - if parts[0] == u.namespace { - port, err := strconv.Atoi(parts[2]) - if err != nil { - u.logger.Debugf("UDP packet process error: %s\n", err) - return - } +// if parts[0] == u.namespace { +// port, err := strconv.Atoi(parts[2]) +// if err != nil { +// u.logger.Debugf("UDP packet process error: %s\n", err) +// return +// } - u.emit("message", parts[1], addr.IP.String(), port) - } -} +// u.emit("message", parts[1], addr.IP.String(), port) +// } +// } func (u *UdpServer) joinMulticastGroup(multicast string, udpConn *net.UDPConn, multicastTTL int, ip string, port int) error { groupAddr, err := net.ResolveUDPAddr("udp4", multicast) @@ -173,49 +182,114 @@ func (u *UdpServer) getAllIPs() []string { func (u *UdpServer) Start() error { if u.opts.Multicast != "" { if u.opts.BindAddress != "" { + u.logger.Debug("Multicast + BindAddress options specified - Binding to a specific interface:", u.opts.BindAddress) // Bind only one interface - return u.startServer(u.opts.BindAddress, u.opts.Port, u.opts.Multicast, u.opts.MulticastTTL) + return u.startServer(u.opts.BindAddress, u.opts.Port, u.opts.Multicast, u.opts.MulticastTTL, []string{u.opts.Multicast}) } //list all interfaces and the ip addresses of each interface + u.logger.Debug("Multicast option specified - listing all interfaces and the ip addresses of each interface") ips := u.getAllIPs() for _, ip := range ips { - err := u.startServer(ip, u.opts.Port, u.opts.Multicast, u.opts.MulticastTTL) + u.logger.Debug("Starting UDP server on IP:", ip, "Port:", u.opts.Port, "Multicast:", u.opts.Multicast, "MulticastTTL:", u.opts.MulticastTTL) + err := u.startServer(ip, u.opts.Port, u.opts.Multicast, u.opts.MulticastTTL, []string{u.opts.Multicast}) if err != nil { u.logger.Error("Error starting server on IP:", ip, err) } } } else if len(u.opts.BroadcastAddrs) > 0 { - return u.startServer(u.opts.BindAddress, u.opts.Port, "", 0) + u.logger.Debug("Starting UDP server on IP (BindAddress):", u.opts.BindAddress, "Port:", u.opts.Port, " BroadcastAddrs option specified - Broadcasting to the specified addresses - BroadcastAddrs:", u.opts.BroadcastAddrs) + return u.startServer(u.opts.BindAddress, u.opts.Port, "", 0, u.opts.BroadcastAddrs) + } else { + broadcastAddrss := u.getBroadcastAddresses() + u.logger.Debug("Starting UDP server on IP (BindAddress):", u.opts.BindAddress, "Port:", u.opts.Port, "No Multicast or BroadcastAddrs options specified - Broadcasting to all interfaces - broadcastAddrss: ", broadcastAddrss) + return u.startServer(u.opts.BindAddress, u.opts.Port, "", 0, broadcastAddrss) } + u.state = "started" + go u.firstDiscoveryMessage() - go u.handleIncomingMessages() + for _, server := range u.servers { + go u.handleIncomingMessagesForServer(server) + } u.startDiscovering() return nil } +func (u *UdpServer) getBroadcastAddresses() []string { + list := []string{} + interfaces, err := net.Interfaces() + if err != nil { + u.logger.Error("Error getting network interfaces:", err) + return list + } + for _, iface := range interfaces { + addrs, err := iface.Addrs() + if err != nil { + u.logger.Error("Error getting addresses for interface:", iface.Name, err) + continue + } + for _, addr := range addrs { + if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + if ipnet.IP.To4() != nil { + // Calculate the broadcast address by inverting the netmask and OR'ing it with the IP address + ip := ipnet.IP.To4() + mask := ipnet.Mask + broadcast := net.IPv4(0, 0, 0, 0) + for i := range ip { + broadcast[i] = ip[i] | ^mask[i] + } + list = append(list, broadcast.String()) + } + } + } + } + return list +} + func (u *UdpServer) firstDiscoveryMessage() { //wait for 1 second before sending the first discovery message time.Sleep(time.Second) u.broadcastDiscoveryMessage() } -func (u *UdpServer) handleIncomingMessages() { +func (u *UdpServer) handleIncomingMessagesForServer(server *UdpServerEntry) { buffer := make([]byte, 2048) - for { - n, addr, err := u.conn.ReadFromUDP(buffer) + for u.state == "started" { + n, addr, err := server.conn.ReadFromUDP(buffer) if err != nil { - u.logger.Println("Error reading from UDP:", err) + u.logger.Errorln("Error reading from UDP:", err) continue } message := string(buffer[:n]) u.logger.Debug("Received message from %s: %s", addr.String(), message) - // Process message here + // Parse the message + parts := strings.Split(message, "|") + if len(parts) != 3 { + u.logger.Debug("Malformed UDP packet received: %s", message) + continue + } + + namespace := parts[0] + + if namespace != u.opts.Namespace { + u.logger.Debug("Message received for a different namespace: %s", namespace) + continue + } + + nodeID := parts[1] + port, err := strconv.Atoi(parts[2]) + if err != nil { + u.logger.Debug("Error parsing port number: %s", err) + continue + } + + u.onUdpMessage(nodeID, addr.IP.String(), port) } + u.logger.Debug("handleIncomingMessagesForServer() stopped") } func (u *UdpServer) startDiscovering() { @@ -228,6 +302,7 @@ func (u *UdpServer) startDiscovering() { for range u.discoverTimer.C { u.broadcastDiscoveryMessage() if u.opts.MaxDiscovery > 0 && u.discoveryCounter >= u.opts.MaxDiscovery { + u.logger.Info("Discovery limit reached, stopping UDP discovery") u.StopDiscovering() } } @@ -235,24 +310,41 @@ func (u *UdpServer) startDiscovering() { } func (u *UdpServer) broadcastDiscoveryMessage() { - message := []byte("discovery message") // Customize your message - destAddr := &net.UDPAddr{IP: net.IPv4bcast, Port: u.opts.Port} - if _, err := u.conn.WriteToUDP(message, destAddr); err != nil { - u.logger.Println("Error broadcasting discovery message:", err) + message := fmt.Sprintf("%s|%s|%d", u.opts.Namespace, u.opts.NodeID, u.opts.Port) + u.logger.Debug("Broadcasting discovery message:", message) + u.discoveryCounter++ + for _, server := range u.servers { + for _, target := range server.discoveryTargets { + destAddr, err := net.ResolveUDPAddr("udp4", target+":"+strconv.Itoa(u.opts.Port)) + if err != nil { + u.logger.Error("Error resolving UDP address:", err) + continue + } + if _, err := server.conn.WriteToUDP([]byte(message), destAddr); err != nil { + u.logger.Error("Error broadcasting discovery message to:", destAddr, " error:", err) + } else { + u.logger.Debug("Discovery message sent to:", destAddr) + } + } } } func (u *UdpServer) StopDiscovering() { + u.logger.Info("Stopping UDP Discovery.") if u.discoverTimer != nil { u.discoverTimer.Stop() u.discoverTimer = nil - u.logger.Info("Discovery limit reached, stopping UDP discovery") } + u.state = "stopped" } func (u *UdpServer) Close() { + u.logger.Info("Closing UDP Server.") + u.state = "stopped" u.StopDiscovering() - if u.conn != nil { - u.conn.Close() + for _, server := range u.servers { + if server.conn != nil { + server.conn.Close() + } } } From f88d4914cebf637e3c907a122d08f2cb535b08a0 Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Wed, 10 Apr 2024 22:09:00 +1200 Subject: [PATCH 08/20] gossip request --- moleculer.go | 9 ++ registry/node.go | 28 +++++- registry/registry.go | 8 +- transit/tcp/gossip.go | 19 ---- transit/tcp/tcp-reader.go | 4 +- transit/tcp/tcp-transporter.go | 167 +++++++++++++++++++++++++++++++-- 6 files changed, 198 insertions(+), 37 deletions(-) delete mode 100644 transit/tcp/gossip.go diff --git a/moleculer.go b/moleculer.go index 016384f..4a6c9a7 100644 --- a/moleculer.go +++ b/moleculer.go @@ -232,6 +232,12 @@ type Node interface { IncreaseSequence() HeartBeat(heartbeat map[string]interface{}) Publish(service map[string]interface{}) + GetUdpAddress() string + GetSequence() int64 + GetCpuSequence() int64 + GetCpu() int64 + IsLocal() bool + Disconnected(isUnexpected bool) } type Options struct { @@ -251,9 +257,12 @@ type Context interface { Meta() Payload } +type ForEachNodeFunc func(node Node) bool type Registry interface { GetNodeByID(nodeID string) Node AddOfflineNode(nodeID, address string, port int) Node + ForEachNode(ForEachNodeFunc) + DisconnectNode(nodeID string) } type BrokerContext interface { diff --git a/registry/node.go b/registry/node.go index d6461ca..8a6381a 100644 --- a/registry/node.go +++ b/registry/node.go @@ -136,6 +136,26 @@ func (node *Node) UpdateInfo(id string, info map[string]interface{}) []map[strin return []map[string]interface{}{} } +func (node *Node) GetSequence() int64 { + return node.sequence +} + +func (node *Node) GetCpuSequence() int64 { + return node.cpuSequence +} + +func (node *Node) GetCpu() int64 { + return node.cpu +} + +func (node *Node) IsLocal() bool { + return node.isLocal +} + +func (node *Node) GetUdpAddress() string { + return node.udpAddress +} + func (node *Node) Update(id string, info map[string]interface{}) (bool, []map[string]interface{}) { if id != node.id { node.logger.Error(fmt.Sprintf("Node.Update() - the id received : %s does not match this node.id : %s", id, node.id)) @@ -208,6 +228,10 @@ func interfaceToString(list []interface{}) []string { return result } +func (node *Node) Disconnected(isUnexpected bool) { + +} + // ExportAsMap export the node info as a map // this map is used to publish the node info to other nodes. func (node *Node) ExportAsMap() map[string]interface{} { @@ -266,10 +290,6 @@ func (node *Node) Available() { node.isAvailable = true } -func (node *Node) IsLocal() bool { - return node.isLocal -} - func (node *Node) IncreaseSequence() { node.sequence++ } diff --git a/registry/registry.go b/registry/registry.go index 9944cce..6703ca4 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -351,8 +351,8 @@ func (registry *ServiceRegistry) removeServicesByNodeID(nodeID string) { registry.events.RemoveByNode(nodeID) } -// disconnectNode remove node info (actions, events) from local registry. -func (registry *ServiceRegistry) disconnectNode(nodeID string) { +// DisconnectNode remove node info (actions, events) from local registry. +func (registry *ServiceRegistry) DisconnectNode(nodeID string) { node, exists := registry.nodes.findNode(nodeID) if !exists { return @@ -366,7 +366,7 @@ func (registry *ServiceRegistry) disconnectNode(nodeID string) { func (registry *ServiceRegistry) checkExpiredRemoteNodes() { expiredNodes := registry.nodes.expiredNodes(registry.heartbeatTimeout) for _, node := range expiredNodes { - registry.disconnectNode(node.GetID()) + registry.DisconnectNode(node.GetID()) } } @@ -420,7 +420,7 @@ func (registry *ServiceRegistry) disconnectMessageReceived(message moleculer.Pay node, exists := registry.nodes.findNode(sender) registry.logger.Debug("disconnectMessageReceived() sender: ", sender, " exists: ", exists) if exists { - registry.disconnectNode(node.GetID()) + registry.DisconnectNode(node.GetID()) } } diff --git a/transit/tcp/gossip.go b/transit/tcp/gossip.go deleted file mode 100644 index a2aeb33..0000000 --- a/transit/tcp/gossip.go +++ /dev/null @@ -1,19 +0,0 @@ -package tcp - -type Gossip struct { -} - -func (g *Gossip) processHello(msgBytes *[]byte) { -} - -// processRequest -func (g *Gossip) processRequest(msgBytes *[]byte) { -} - -// processResponse -func (g *Gossip) processResponse(msgBytes *[]byte) { -} - -func isGossipMessage(msgType byte) bool { - return msgType == PACKET_GOSSIP_REQ || msgType == PACKET_GOSSIP_RES || msgType == PACKET_GOSSIP_HELLO -} diff --git a/transit/tcp/tcp-reader.go b/transit/tcp/tcp-reader.go index 681c98e..1d14016 100644 --- a/transit/tcp/tcp-reader.go +++ b/transit/tcp/tcp-reader.go @@ -16,7 +16,7 @@ const ( CLOSED ) -type OnMessageFunc func(msgType int, msgBytes *[]byte) +type OnMessageFunc func(fromAddrss string, msgType int, msgBytes *[]byte) type TcpReader struct { port int @@ -77,7 +77,7 @@ func (r *TcpReader) handleConnection(conn net.Conn) { r.logger.Errorf("Error reading message from '%s': %s", address, err) break } - r.onMessage(msgType, &msgBytes) + r.onMessage(address, msgType, &msgBytes) } r.closeSocket(conn) diff --git a/transit/tcp/tcp-transporter.go b/transit/tcp/tcp-transporter.go index 3e3b4e0..1092dea 100644 --- a/transit/tcp/tcp-transporter.go +++ b/transit/tcp/tcp-transporter.go @@ -4,6 +4,7 @@ import ( "time" "github.com/moleculer-go/moleculer" + payloadPkg "github.com/moleculer-go/moleculer/payload" "github.com/moleculer-go/moleculer/serializer" "github.com/moleculer-go/moleculer/transit" @@ -15,7 +16,6 @@ type TCPTransporter struct { tcpReader *TcpReader tcpWriter *TcpWriter udpServer *UdpServer - gossip *Gossip registry moleculer.Registry logger *log.Entry @@ -101,20 +101,169 @@ const ( PACKET_GOSSIP_HELLO = 8 ) -func (transporter *TCPTransporter) onTcpMessage(msgType int, msgBytes *[]byte) { +func (transporter *TCPTransporter) onTcpMessage(fromAddrss string, msgType int, msgBytes *[]byte) { switch msgType { case PACKET_GOSSIP_HELLO: - transporter.gossip.processHello(msgBytes) + transporter.onGossipHello(fromAddrss, msgBytes) case PACKET_GOSSIP_REQ: - transporter.gossip.processRequest(msgBytes) + transporter.onGossipRequest(msgBytes) case PACKET_GOSSIP_RES: - transporter.gossip.processResponse(msgBytes) + transporter.onGossipResponse(msgBytes) default: transporter.incomingMessage(msgType, msgBytes) } } -func (transporter *TCPTransporter) msgTypeToCommand(msgType int) string { +func (transporter *TCPTransporter) onGossipHello(fromAddrss string, msgBytes *[]byte) { + packet := transporter.serializer.BytesToPayload(msgBytes) + payload := packet.Get("payload") + nodeID := payload.Get("sender").String() + + node := transporter.registry.GetNodeByID(nodeID) + if node == nil { + // Unknown node. Register as offline node + node = transporter.registry.AddOfflineNode(nodeID, payload.Get("host").String(), payload.Get("port").Int()) + } + if node.GetUdpAddress() == "" { + node.UpdateInfo(nodeID, map[string]interface{}{ + "udpAddress": fromAddrss, + }) + } +} + +// processRequest +func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { + packet := transporter.serializer.BytesToPayload(msgBytes) + payload := packet.Get("payload") + + onlineResponse := map[string]interface{}{} + offlineResponse := map[string]interface{}{} + + transporter.registry.ForEachNode(func(node moleculer.Node) bool { + + onlineMap := payload.Get("online") + offlineMap := payload.Get("offline") + var seq int64 = 0 + var cpuSeq int64 = 0 + var cpu int64 = 0 + var offline moleculer.Payload + var online moleculer.Payload + + if offlineMap.Exists() { + offline = offlineMap.Get(node.GetID()) + if offline.Exists() { + transporter.logger.Debug("received seq for " + node.GetID()) + seq = offline.Int64() + } + } + if onlineMap.Exists() { + online = onlineMap.Get(node.GetID()) + if online.Exists() { + transporter.logger.Debug("received seq, cpuSeq, cpu for " + node.GetID()) + seq = online.Get("seq").Int64() + cpuSeq = online.Get("cpuSeq").Int64() + cpu = online.Get("cpu").Int64() + } + } + + if seq != 0 && seq < node.GetSequence() { + transporter.logger.Debug("We have newer info or requester doesn't know it") + if node.IsAvailable() { + info := node.ExportAsMap() + onlineResponse[node.GetID()] = []interface{}{info, node.GetCpuSequence(), node.GetCpu()} + transporter.logger.Debug("Node is available - send back the node info and cpu, cpuSed to " + node.GetID()) + } else { + offlineResponse[node.GetID()] = node.GetSequence() + transporter.logger.Debug("Node is offline - send back the seq to " + node.GetID()) + } + return true + } + + if offline != nil && offline.Exists() { + transporter.logger.Debug("Requester said it is OFFLINE") + if !node.IsAvailable() { + transporter.logger.Debug("We also know it as offline - update the seq") + if seq > node.GetSequence() { + node.UpdateInfo(node.GetID(), map[string]interface{}{ + "seq": seq, + }) + } + return true + } + + if !node.IsLocal() { + transporter.logger.Debug("our current state for it is online - change it to offline and update seq - nodeID:", node.GetID(), "seq:", seq) + // We know it is online, so we change it to offline + transporter.registry.DisconnectNode(node.GetID()) + + // Update the 'seq' to the received value + node.UpdateInfo(node.GetID(), map[string]interface{}{ + "seq": seq, + }) + return true + } + + if node.IsLocal() { + transporter.logger.Debug("msg is about the Local node - update the seq and send back info, cpu and cpuSeq") + // Update the 'seq' to the received value + node.UpdateInfo(node.GetID(), map[string]interface{}{ + "seq": seq + 1, + }) + onlineResponse[node.GetID()] = []interface{}{node.ExportAsMap(), node.GetCpuSequence(), node.GetCpu()} + + return true + } + + } + + if online != nil && online.Exists() { + // Requester said it is ONLINE + if node.IsAvailable() { + if cpuSeq > node.GetCpuSequence() { + // We update CPU info + node.UpdateInfo(node.GetID(), map[string]interface{}{ + "cpu": cpu, + "cpuSeq": cpuSeq, + }) + transporter.logger.Debug("CPU info updated for " + node.GetID()) + } else if cpuSeq < node.GetCpuSequence() { + // We have newer info, send back + //TODO check where we process this to see if we handle this array correctly + onlineResponse[node.GetID()] = []interface{}{node.GetCpuSequence(), node.GetCpu()} + transporter.logger.Debug("CPU info sent back to " + node.GetID()) + } + } else { + // We know it as offline. We do nothing, because we'll request it and we'll receive its INFO. + return true + } + } + + return true + }) + + if len(onlineResponse) > 0 || len(offlineResponse) > 0 { + // Send back the Gossip response to the sender + sender := payload.Get("sender").String() + + // Send back the Gossip response to the sender + transporter.Publish(msgTypeToCommand(PACKET_GOSSIP_RES), sender, payloadPkg.Empty().Add("online", onlineResponse).Add("offline", offlineResponse)) + + transporter.logger.Debug("Gossip response sent to " + sender) + } else { + transporter.logger.Debug("No response sent to " + payload.Get("sender").String()) + } + +} + +// processResponse +func (transporter *TCPTransporter) onGossipResponse(msgBytes *[]byte) { +} + +func isGossipMessage(msgType byte) bool { + return msgType == PACKET_GOSSIP_REQ || msgType == PACKET_GOSSIP_RES || msgType == PACKET_GOSSIP_HELLO +} + +func msgTypeToCommand(msgType int) string { switch msgType { case PACKET_EVENT: return "EVENT" @@ -146,7 +295,7 @@ func (transporter *TCPTransporter) msgTypeToCommand(msgType int) string { } func (transporter *TCPTransporter) incomingMessage(msgType int, msgBytes *[]byte) { - command := transporter.msgTypeToCommand(msgType) + command := msgTypeToCommand(msgType) message := transporter.serializer.BytesToPayload(msgBytes) if transporter.validateMsg(message) { if handlers, ok := transporter.handlers[command]; ok { @@ -205,9 +354,11 @@ func addIpToList(ipList []string, address string) []string { return ipList } +// TODO - check full lifecycle - this message creates or updates a node with ip address and port to connect to directly +// need to find where the TCP connection step happens.. is not happening here - where is this node info used ? func (transporter *TCPTransporter) onUdpMessage(nodeID, address string, port int) { if nodeID != "" && nodeID != transporter.options.NodeId { - transporter.logger.Debug(`UDP discovery received from ${address} on ${nodeID}.`) + transporter.logger.Debug("UDP discovery received from " + address + " nodeId: " + nodeID + " port: " + string(port)) node := transporter.registry.GetNodeByID(nodeID) if node == nil { // Unknown node. Register as offline node From db68c1475425b3cc81ea75c375bf2012cfb2b91f Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Wed, 10 Apr 2024 22:46:08 +1200 Subject: [PATCH 09/20] gossip response --- moleculer.go | 1 + registry/registry.go | 4 +- transit/tcp/tcp-transporter.go | 97 ++++++++++++++++++++++++++++++---- 3 files changed, 91 insertions(+), 11 deletions(-) diff --git a/moleculer.go b/moleculer.go index 4a6c9a7..7a6c80b 100644 --- a/moleculer.go +++ b/moleculer.go @@ -263,6 +263,7 @@ type Registry interface { AddOfflineNode(nodeID, address string, port int) Node ForEachNode(ForEachNodeFunc) DisconnectNode(nodeID string) + RemoteNodeInfoReceived(message Payload) } type BrokerContext interface { diff --git a/registry/registry.go b/registry/registry.go index 6703ca4..9516b56 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -129,7 +129,7 @@ func (registry *ServiceRegistry) setupMessageHandlers() { messageHandler := map[string]messageHandlerFunc{ "HEARTBEAT": registry.filterMessages(registry.heartbeatMessageReceived), "DISCONNECT": registry.filterMessages(registry.disconnectMessageReceived), - "INFO": registry.filterMessages(registry.remoteNodeInfoReceived), + "INFO": registry.filterMessages(registry.RemoteNodeInfoReceived), } registry.broker.Bus().On("$registry.transit.message", func(args ...interface{}) { registry.logger.Trace("Registry -> $registry.transit.message event - args: ", args) @@ -433,7 +433,7 @@ func compatibility(info map[string]interface{}) map[string]interface{} { } // remoteNodeInfoReceived process the remote node info message and add to local registry. -func (registry *ServiceRegistry) remoteNodeInfoReceived(message moleculer.Payload) { +func (registry *ServiceRegistry) RemoteNodeInfoReceived(message moleculer.Payload) { registry.nodeReceivedMutex.Lock() defer registry.nodeReceivedMutex.Unlock() diff --git a/transit/tcp/tcp-transporter.go b/transit/tcp/tcp-transporter.go index 1092dea..e4417fb 100644 --- a/transit/tcp/tcp-transporter.go +++ b/transit/tcp/tcp-transporter.go @@ -131,7 +131,6 @@ func (transporter *TCPTransporter) onGossipHello(fromAddrss string, msgBytes *[] } } -// processRequest func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { packet := transporter.serializer.BytesToPayload(msgBytes) payload := packet.Get("payload") @@ -210,10 +209,8 @@ func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { "seq": seq + 1, }) onlineResponse[node.GetID()] = []interface{}{node.ExportAsMap(), node.GetCpuSequence(), node.GetCpu()} - return true } - } if online != nil && online.Exists() { @@ -237,17 +234,12 @@ func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { return true } } - return true }) if len(onlineResponse) > 0 || len(offlineResponse) > 0 { - // Send back the Gossip response to the sender sender := payload.Get("sender").String() - - // Send back the Gossip response to the sender transporter.Publish(msgTypeToCommand(PACKET_GOSSIP_RES), sender, payloadPkg.Empty().Add("online", onlineResponse).Add("offline", offlineResponse)) - transporter.logger.Debug("Gossip response sent to " + sender) } else { transporter.logger.Debug("No response sent to " + payload.Get("sender").String()) @@ -255,8 +247,95 @@ func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { } -// processResponse func (transporter *TCPTransporter) onGossipResponse(msgBytes *[]byte) { + packet := transporter.serializer.BytesToPayload(msgBytes) + payload := packet.Get("payload") + sender := payload.Get("sender").String() + + online := payload.Get("online") + offline := payload.Get("offline") + + if online.Exists() { + transporter.logger.Debug("Received online info from nodeID: " + sender) + online.ForEach(func(key interface{}, value moleculer.Payload) bool { + nodeID, ok := key.(string) + if !ok { + transporter.logger.Error("Error parsing online nodeID") + return true + } + node := transporter.registry.GetNodeByID(nodeID) + if node != nil && node.IsLocal() { + transporter.logger.Debug("Received info about the local node - ignore it") + return true + } + row := online.Get(nodeID).Array() + info, cpu, cpuSeq := parseGossipResponse(row) + + if info != nil && (node != nil || node.GetSequence() < info["seq"].(int64)) { + transporter.logger.Debug("If we don't know it, or know, but has smaller seq, update 'info'") + info["sender"] = sender + transporter.registry.RemoteNodeInfoReceived(payloadPkg.New(info)) + } + if node != nil && cpuSeq > node.GetCpuSequence() { + transporter.logger.Debug("If we know it and has smaller cpuSeq, update 'cpu'") + node.HeartBeat(map[string]interface{}{ + "cpu": cpu, + "cpuSeq": cpuSeq, + }) + } + return true + }) + } + + if offline.Exists() { + transporter.logger.Debug("Received offline info from nodeID: " + sender) + offline.ForEach(func(key interface{}, value moleculer.Payload) bool { + nodeID, ok := key.(string) + if !ok { + transporter.logger.Error("Error parsing offline nodeID") + return true + } + node := transporter.registry.GetNodeByID(nodeID) + if node != nil && node.IsLocal() { + transporter.logger.Debug("Received info about the local node - ignore it") + return true + } + if node == nil { + return true + } + + seq := offline.Get(nodeID).Int64() + + if node.GetSequence() < seq { + if node.IsAvailable() { + transporter.logger.Debug("Node is online, will change it to offline") + transporter.registry.DisconnectNode(nodeID) + } + node.UpdateInfo(nodeID, map[string]interface{}{ + "seq": seq, + }) + } + return true + }) + } +} + +func parseGossipResponse(row []moleculer.Payload) (info map[string]interface{}, cpu int64, cpuSeq int64) { + cpuSeq = -1 + cpu = -1 + if len(row) == 1 { + info = row[0].RawMap() + } + if len(row) == 2 { + cpuSeq = row[0].Int64() + cpu = row[1].Int64() + } + if len(row) == 3 { + info = row[0].RawMap() + cpuSeq = row[1].Int64() + cpu = row[2].Int64() + } + return info, cpu, cpuSeq } func isGossipMessage(msgType byte) bool { From 155b79436d7d10eaf9952acf2a3f0d167f5760df Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Thu, 11 Apr 2024 15:25:12 +1200 Subject: [PATCH 10/20] gossip protocol + final wiring --- go.mod | 5 +- go.sum | 15 +- moleculer.go | 4 + registry/node.go | 9 + registry/nodeCatalog.go | 6 + registry/registry.go | 81 +++++---- transit/pubsub/pubsub.go | 13 +- transit/tcp/gossip.go | 306 +++++++++++++++++++++++++++++++++ transit/tcp/tcp-reader.go | 10 +- transit/tcp/tcp-transporter.go | 297 ++++++-------------------------- transit/tcp/tcp-writer.go | 2 +- transit/tcp/udp.go | 13 +- util/metrics.go | 50 ++++++ 13 files changed, 518 insertions(+), 293 deletions(-) create mode 100644 transit/tcp/gossip.go create mode 100644 util/metrics.go diff --git a/go.mod b/go.mod index d37e112..c27482c 100644 --- a/go.mod +++ b/go.mod @@ -22,13 +22,16 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/procfs v0.0.0-20190503130316-740c07785007 // indirect github.com/segmentio/kafka-go v0.4.18 + github.com/shirou/gopsutil v3.21.11+incompatible github.com/sirupsen/logrus v1.4.2 github.com/spf13/cobra v0.0.3 github.com/spf13/viper v1.3.2 github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 github.com/tidwall/gjson v1.9.3 github.com/tidwall/sjson v1.0.4 + github.com/tklauser/go-sysconf v0.3.13 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.3.2 // indirect go.mongodb.org/mongo-driver v1.5.2 - golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 // indirect + golang.org/x/net v0.0.0-20201021035429-f5854403a974 ) diff --git a/go.sum b/go.sum index 5632cc8..2c85f33 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,8 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= @@ -167,6 +169,8 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/segmentio/kafka-go v0.4.18 h1:/LwffTZgFnfjgkUu1ZzHTwJJ39vW77wwUA0J6ftBbGw= github.com/segmentio/kafka-go v0.4.18/go.mod h1:19+Eg7KwrNKy/PFhiIthEPkO8k+ac7/ZYXwYM9Df10w= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= @@ -201,6 +205,10 @@ github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.0.4 h1:UcdIRXff12Lpnu3OLtZvnc03g4vH2suXDXhBwBqmzYg= github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= +github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= +github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= +github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= +github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -213,6 +221,8 @@ github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mongodb.org/mongo-driver v1.5.2 h1:AsxOLoJTgP6YNM0fXWw4OjdluYmWzQYp+lFJL7xu9fU= @@ -254,13 +264,14 @@ golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 h1:cdsMqa2nXzqlgs183pHxtvoVwU7CyzaCTAUOg94af4c= -golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= diff --git a/moleculer.go b/moleculer.go index 7a6c80b..f357916 100644 --- a/moleculer.go +++ b/moleculer.go @@ -123,6 +123,7 @@ type Config struct { Transporter string TransporterFactory TransporterFactoryFunc StrategyFactory StrategyFactoryFunc + UpdateNodeMetricsFrequency time.Duration HeartbeatFrequency time.Duration HeartbeatTimeout time.Duration OfflineCheckFrequency time.Duration @@ -153,6 +154,7 @@ var DefaultConfig = Config{ LogFormat: "TEXT", DiscoverNodeID: discoverNodeID, Transporter: "MEMORY", + UpdateNodeMetricsFrequency: 5 * time.Second, HeartbeatFrequency: 5 * time.Second, HeartbeatTimeout: 15 * time.Second, OfflineCheckFrequency: 20 * time.Second, @@ -238,6 +240,7 @@ type Node interface { GetCpu() int64 IsLocal() bool Disconnected(isUnexpected bool) + UpdateMetrics() } type Options struct { @@ -264,6 +267,7 @@ type Registry interface { ForEachNode(ForEachNodeFunc) DisconnectNode(nodeID string) RemoteNodeInfoReceived(message Payload) + GetLocalNode() Node } type BrokerContext interface { diff --git a/registry/node.go b/registry/node.go index 8a6381a..b236398 100644 --- a/registry/node.go +++ b/registry/node.go @@ -170,6 +170,15 @@ func (node *Node) Update(id string, info map[string]interface{}) (bool, []map[st return reconnected, removedServices } +func (node *Node) UpdateMetrics() { + cpu, err := util.GetCpuUsage() + if err != nil { + node.logger.Error("Error getting cpu usage:", err) + return + } + node.cpu = int64(cpu) +} + // FilterServices return all services excluding local services (example: $node) and return all removed services, services that existed in the currentServices list but // no longer exist in th new list coming from the info package func FilterServices(currentServices []map[string]interface{}, info map[string]interface{}) ([]map[string]interface{}, []map[string]interface{}) { diff --git a/registry/nodeCatalog.go b/registry/nodeCatalog.go index 769823b..7a9bb2b 100644 --- a/registry/nodeCatalog.go +++ b/registry/nodeCatalog.go @@ -31,6 +31,12 @@ func (catalog *NodeCatalog) HeartBeat(heartbeat map[string]interface{}) bool { return false } +func (catalog *NodeCatalog) ForEachNode(forEAchFunc moleculer.ForEachNodeFunc) { + catalog.nodes.Range(func(key, value interface{}) bool { + return forEAchFunc(value.(moleculer.Node)) + }) +} + func (catalog *NodeCatalog) list() []moleculer.Node { var result []moleculer.Node catalog.nodes.Range(func(key, value interface{}) bool { diff --git a/registry/registry.go b/registry/registry.go index 9516b56..15bda07 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -24,22 +24,23 @@ import ( type messageHandlerFunc func(message moleculer.Payload) type ServiceRegistry struct { - logger *log.Entry - transit transit.Transit - localNode moleculer.Node - nodes *NodeCatalog - services *ServiceCatalog - actions *ActionCatalog - events *EventCatalog - broker *moleculer.BrokerDelegates - strategy strategy.Strategy - stopping bool - heartbeatFrequency time.Duration - heartbeatTimeout time.Duration - offlineCheckFrequency time.Duration - offlineTimeout time.Duration - nodeReceivedMutex *sync.Mutex - namespace string + logger *log.Entry + transit transit.Transit + localNode moleculer.Node + nodes *NodeCatalog + services *ServiceCatalog + actions *ActionCatalog + events *EventCatalog + broker *moleculer.BrokerDelegates + strategy strategy.Strategy + stopping bool + updateNodeMetricsFrequency time.Duration + heartbeatFrequency time.Duration + heartbeatTimeout time.Duration + offlineCheckFrequency time.Duration + offlineTimeout time.Duration + nodeReceivedMutex *sync.Mutex + namespace string } // createTransit create a transit instance based on the config. @@ -66,22 +67,23 @@ func CreateRegistry(nodeID string, broker *moleculer.BrokerDelegates) *ServiceRe localNode := CreateNode(nodeID, true, logger.WithField("Node", nodeID)) localNode.Unavailable() registry := &ServiceRegistry{ - broker: broker, - transit: transit, - strategy: strategy, - logger: logger, - localNode: localNode, - actions: CreateActionCatalog(logger.WithField("catalog", "Actions")), - events: CreateEventCatalog(logger.WithField("catalog", "Events")), - services: CreateServiceCatalog(logger.WithField("catalog", "Services")), - nodes: CreateNodesCatalog(logger.WithField("catalog", "Nodes")), - heartbeatFrequency: config.HeartbeatFrequency, - heartbeatTimeout: config.HeartbeatTimeout, - offlineCheckFrequency: config.OfflineCheckFrequency, - offlineTimeout: config.OfflineTimeout, - stopping: false, - nodeReceivedMutex: &sync.Mutex{}, - namespace: config.Namespace, + broker: broker, + transit: transit, + strategy: strategy, + logger: logger, + localNode: localNode, + actions: CreateActionCatalog(logger.WithField("catalog", "Actions")), + events: CreateEventCatalog(logger.WithField("catalog", "Events")), + services: CreateServiceCatalog(logger.WithField("catalog", "Services")), + nodes: CreateNodesCatalog(logger.WithField("catalog", "Nodes")), + updateNodeMetricsFrequency: config.UpdateNodeMetricsFrequency, + heartbeatFrequency: config.HeartbeatFrequency, + heartbeatTimeout: config.HeartbeatTimeout, + offlineCheckFrequency: config.OfflineCheckFrequency, + offlineTimeout: config.OfflineTimeout, + stopping: false, + nodeReceivedMutex: &sync.Mutex{}, + namespace: config.Namespace, } registry.logger.Debug("Service Registry created for broker: ", nodeID) @@ -164,6 +166,11 @@ func (registry *ServiceRegistry) GetNodeByID(nodeID string) moleculer.Node { return node } +func (registry *ServiceRegistry) heartbeat() { + registry.localNode.UpdateMetrics() + registry.transit.SendHeartbeat() +} + // Start : start the registry background processes. func (registry *ServiceRegistry) Start() { registry.logger.Debug("Registry Start() ") @@ -177,7 +184,7 @@ func (registry *ServiceRegistry) Start() { registry.nodes.Add(registry.localNode) if registry.heartbeatFrequency > 0 { - go registry.loopWhileAlive(registry.heartbeatFrequency, registry.transit.SendHeartbeat) + go registry.loopWhileAlive(registry.heartbeatFrequency, registry.heartbeat) } if registry.heartbeatTimeout > 0 { go registry.loopWhileAlive(registry.heartbeatTimeout, registry.checkExpiredRemoteNodes) @@ -432,6 +439,14 @@ func compatibility(info map[string]interface{}) map[string]interface{} { return info } +func (registry *ServiceRegistry) ForEachNode(forEAchFunc moleculer.ForEachNodeFunc) { + registry.nodes.ForEachNode(forEAchFunc) +} + +func (registry *ServiceRegistry) GetLocalNode() moleculer.Node { + return registry.localNode +} + // remoteNodeInfoReceived process the remote node info message and add to local registry. func (registry *ServiceRegistry) RemoteNodeInfoReceived(message moleculer.Payload) { registry.nodeReceivedMutex.Lock() diff --git a/transit/pubsub/pubsub.go b/transit/pubsub/pubsub.go index 45ad532..689daf5 100644 --- a/transit/pubsub/pubsub.go +++ b/transit/pubsub/pubsub.go @@ -352,9 +352,11 @@ func (pubsub *PubSub) SendHeartbeat() { "ver": version.MoleculerProtocol(), } message, err := pubsub.serializer.MapToPayload(&payload) - if err == nil { - pubsub.transport.Publish("HEARTBEAT", "", message) + if err != nil { + pubsub.logger.Error("SendHeartbeat() Error serializing the payload: ", payload, " error: ", err) + return } + pubsub.transport.Publish("HEARTBEAT", "", message) } func (pubsub *PubSub) DiscoverNode(nodeID string) { @@ -363,9 +365,12 @@ func (pubsub *PubSub) DiscoverNode(nodeID string) { "ver": version.MoleculerProtocol(), } message, err := pubsub.serializer.MapToPayload(&payload) - if err == nil { - pubsub.transport.Publish("DISCOVER", nodeID, message) + if err != nil { + pubsub.logger.Error("DiscoverNode() Error serializing the payload: ", payload, " error: ", err) + return } + pubsub.transport.Publish("DISCOVER", nodeID, message) + } // Emit emit an event to all services that listens to this event. diff --git a/transit/tcp/gossip.go b/transit/tcp/gossip.go new file mode 100644 index 0000000..7ed38bd --- /dev/null +++ b/transit/tcp/gossip.go @@ -0,0 +1,306 @@ +package tcp + +import ( + "math/rand" + "time" + + "github.com/moleculer-go/moleculer" + payloadPkg "github.com/moleculer-go/moleculer/payload" +) + +func (transporter *TCPTransporter) startGossipTimer() { + transporter.gossipTimer = time.NewTicker(time.Second * time.Duration(transporter.options.GossipPeriod)) + go func() { + for range transporter.gossipTimer.C { + transporter.sendGossipRequest() + } + }() +} + +func (transporter *TCPTransporter) sendGossipRequest() { + + transporter.logger.Debug("Sending gossip request") + + node := transporter.registry.GetLocalNode() + node.UpdateMetrics() + + onlineResponse := map[string]interface{}{} + offlineResponse := map[string]interface{}{} + onlineNodes := []moleculer.Node{} + offlineNodes := []moleculer.Node{} + + transporter.registry.ForEachNode(func(node moleculer.Node) bool { + if node.IsAvailable() { + onlineResponse[node.GetID()] = []interface{}{node.GetSequence(), node.GetCpuSequence(), node.GetCpu()} + onlineNodes = append(onlineNodes, node) + } else { + offlineResponse[node.GetID()] = node.GetSequence() + offlineNodes = append(offlineNodes, node) + } + return true + }) + + payload := payloadPkg.Empty() + packet := payloadPkg.Empty() + packet.Add("payload", payload) + if len(onlineResponse) > 0 { + payload.Add("online", onlineResponse) + } + if len(offlineResponse) > 0 { + payload.Add("offline", offlineResponse) + } + + if len(onlineResponse) > 0 { + transporter.sendGossipToRandomEndpoint(packet, onlineNodes) + } + + if len(offlineNodes) > 0 { + ratio := float64(len(offlineNodes)) / float64(len(onlineNodes)+1) + if ratio >= 1 || rand.Float64() < ratio { + transporter.sendGossipToRandomEndpoint(packet, offlineNodes) + } + } +} + +func (transporter *TCPTransporter) sendGossipToRandomEndpoint(packet moleculer.Payload, nodes []moleculer.Node) { + if len(nodes) == 0 { + return + } + node := nodes[rand.Intn(len(nodes))] + transporter.logger.Debug("Sending gossip request to " + node.GetID()) + transporter.Publish(msgTypeToCommand(PACKET_GOSSIP_REQ), node.GetID(), packet) +} + +func (transporter *TCPTransporter) onGossipHello(fromAddrss string, msgBytes *[]byte) { + packet := transporter.serializer.BytesToPayload(msgBytes) + payload := packet.Get("payload") + sender := payload.Get("sender").String() + + transporter.logger.Debug("Received gossip hello from " + sender) + + node := transporter.registry.GetNodeByID(sender) + if node == nil { + // Unknown node. Register as offline node + node = transporter.registry.AddOfflineNode(sender, payload.Get("host").String(), payload.Get("port").Int()) + } + if node.GetUdpAddress() == "" { + node.UpdateInfo(sender, map[string]interface{}{ + "udpAddress": fromAddrss, + }) + } +} + +func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { + packet := transporter.serializer.BytesToPayload(msgBytes) + payload := packet.Get("payload") + sender := payload.Get("sender").String() + + transporter.logger.Debug("Received gossip request from " + sender) + + onlineResponse := map[string]interface{}{} + offlineResponse := map[string]interface{}{} + + transporter.registry.ForEachNode(func(node moleculer.Node) bool { + + onlineMap := payload.Get("online") + offlineMap := payload.Get("offline") + var seq int64 = 0 + var cpuSeq int64 = 0 + var cpu int64 = 0 + var offline moleculer.Payload + var online moleculer.Payload + + if offlineMap.Exists() { + offline = offlineMap.Get(node.GetID()) + if offline.Exists() { + transporter.logger.Debug("received seq for " + node.GetID()) + seq = offline.Int64() + } + } + if onlineMap.Exists() { + online = onlineMap.Get(node.GetID()) + if online.Exists() { + transporter.logger.Debug("received seq, cpuSeq, cpu for " + node.GetID()) + seq = online.Get("seq").Int64() + cpuSeq = online.Get("cpuSeq").Int64() + cpu = online.Get("cpu").Int64() + } + } + + if seq != 0 && seq < node.GetSequence() { + transporter.logger.Debug("We have newer info or requester doesn't know it") + if node.IsAvailable() { + info := node.ExportAsMap() + onlineResponse[node.GetID()] = []interface{}{info, node.GetCpuSequence(), node.GetCpu()} + transporter.logger.Debug("Node is available - send back the node info and cpu, cpuSed to " + node.GetID()) + } else { + offlineResponse[node.GetID()] = node.GetSequence() + transporter.logger.Debug("Node is offline - send back the seq to " + node.GetID()) + } + return true + } + + if offline != nil && offline.Exists() { + transporter.logger.Debug("Requester said it is OFFLINE") + if !node.IsAvailable() { + transporter.logger.Debug("We also know it as offline - update the seq") + if seq > node.GetSequence() { + node.UpdateInfo(node.GetID(), map[string]interface{}{ + "seq": seq, + }) + } + return true + } + + if !node.IsLocal() { + transporter.logger.Debug("our current state for it is online - change it to offline and update seq - nodeID:", node.GetID(), "seq:", seq) + // We know it is online, so we change it to offline + transporter.registry.DisconnectNode(node.GetID()) + + // Update the 'seq' to the received value + node.UpdateInfo(node.GetID(), map[string]interface{}{ + "seq": seq, + }) + return true + } + + if node.IsLocal() { + transporter.logger.Debug("msg is about the Local node - update the seq and send back info, cpu and cpuSeq") + // Update the 'seq' to the received value + node.UpdateInfo(node.GetID(), map[string]interface{}{ + "seq": seq + 1, + }) + onlineResponse[node.GetID()] = []interface{}{node.ExportAsMap(), node.GetCpuSequence(), node.GetCpu()} + return true + } + } + + if online != nil && online.Exists() { + // Requester said it is ONLINE + if node.IsAvailable() { + if cpuSeq > node.GetCpuSequence() { + // We update CPU info + node.UpdateInfo(node.GetID(), map[string]interface{}{ + "cpu": cpu, + "cpuSeq": cpuSeq, + }) + transporter.logger.Debug("CPU info updated for " + node.GetID()) + } else if cpuSeq < node.GetCpuSequence() { + // We have newer info, send back + onlineResponse[node.GetID()] = []interface{}{node.GetCpuSequence(), node.GetCpu()} + transporter.logger.Debug("CPU info sent back to " + node.GetID()) + } + } else { + // We know it as offline. We do nothing, because we'll request it and we'll receive its INFO. + return true + } + } + return true + }) + + if len(onlineResponse) > 0 || len(offlineResponse) > 0 { + sender := payload.Get("sender").String() + transporter.Publish(msgTypeToCommand(PACKET_GOSSIP_RES), sender, payloadPkg.Empty().Add("online", onlineResponse).Add("offline", offlineResponse)) + transporter.logger.Debug("Gossip response sent to " + sender) + } else { + transporter.logger.Debug("No response sent to " + payload.Get("sender").String()) + } + +} + +func (transporter *TCPTransporter) onGossipResponse(msgBytes *[]byte) { + packet := transporter.serializer.BytesToPayload(msgBytes) + payload := packet.Get("payload") + sender := payload.Get("sender").String() + + transporter.logger.Debug("Received gossip response from " + sender) + + online := payload.Get("online") + offline := payload.Get("offline") + + if online.Exists() { + transporter.logger.Debug("Received online info from nodeID: " + sender) + online.ForEach(func(key interface{}, value moleculer.Payload) bool { + nodeID, ok := key.(string) + if !ok { + transporter.logger.Error("Error parsing online nodeID") + return true + } + node := transporter.registry.GetNodeByID(nodeID) + if node != nil && node.IsLocal() { + transporter.logger.Debug("Received info about the local node - ignore it") + return true + } + row := online.Get(nodeID).Array() + info, cpu, cpuSeq := parseGossipResponse(row) + + if info != nil && (node != nil || node.GetSequence() < info["seq"].(int64)) { + transporter.logger.Debug("If we don't know it, or know, but has smaller seq, update 'info'") + info["sender"] = sender + transporter.registry.RemoteNodeInfoReceived(payloadPkg.New(info)) + } + if node != nil && cpuSeq > node.GetCpuSequence() { + transporter.logger.Debug("If we know it and has smaller cpuSeq, update 'cpu'") + node.HeartBeat(map[string]interface{}{ + "cpu": cpu, + "cpuSeq": cpuSeq, + }) + } + return true + }) + } + + if offline.Exists() { + transporter.logger.Debug("Received offline info from nodeID: " + sender) + offline.ForEach(func(key interface{}, value moleculer.Payload) bool { + nodeID, ok := key.(string) + if !ok { + transporter.logger.Error("Error parsing offline nodeID") + return true + } + node := transporter.registry.GetNodeByID(nodeID) + if node != nil && node.IsLocal() { + transporter.logger.Debug("Received info about the local node - ignore it") + return true + } + if node == nil { + return true + } + + seq := offline.Get(nodeID).Int64() + + if node.GetSequence() < seq { + if node.IsAvailable() { + transporter.logger.Debug("Node is online, will change it to offline") + transporter.registry.DisconnectNode(nodeID) + } + node.UpdateInfo(nodeID, map[string]interface{}{ + "seq": seq, + }) + } + return true + }) + } +} + +func parseGossipResponse(row []moleculer.Payload) (info map[string]interface{}, cpu int64, cpuSeq int64) { + cpuSeq = -1 + cpu = -1 + if len(row) == 1 { + info = row[0].RawMap() + } + if len(row) == 2 { + cpuSeq = row[0].Int64() + cpu = row[1].Int64() + } + if len(row) == 3 { + info = row[0].RawMap() + cpuSeq = row[1].Int64() + cpu = row[2].Int64() + } + return info, cpu, cpuSeq +} + +func isGossipMessage(msgType byte) bool { + return msgType == PACKET_GOSSIP_REQ || msgType == PACKET_GOSSIP_RES || msgType == PACKET_GOSSIP_HELLO +} diff --git a/transit/tcp/tcp-reader.go b/transit/tcp/tcp-reader.go index 1d14016..9dc66b6 100644 --- a/transit/tcp/tcp-reader.go +++ b/transit/tcp/tcp-reader.go @@ -12,8 +12,8 @@ import ( type State int const ( - LISTENING State = iota - CLOSED + STARTED State = iota + STOPPED ) type OnMessageFunc func(fromAddrss string, msgType int, msgBytes *[]byte) @@ -47,9 +47,9 @@ func (r *TcpReader) Listen() { r.logger.Infof("TCP server is listening on port %d", r.port) - r.state = LISTENING + r.state = STARTED go func() { - for r.state == LISTENING { + for r.state == STARTED { conn, err := r.listener.Accept() if err != nil { r.logger.Error("Error accepting connection: ", err) @@ -139,7 +139,7 @@ func (r *TcpReader) closeSocket(conn net.Conn) { } func (r *TcpReader) Close() { - r.state = CLOSED + r.state = STOPPED r.listener.Close() for conn := range r.sockets { r.closeSocket(conn) diff --git a/transit/tcp/tcp-transporter.go b/transit/tcp/tcp-transporter.go index e4417fb..e64a94e 100644 --- a/transit/tcp/tcp-transporter.go +++ b/transit/tcp/tcp-transporter.go @@ -4,7 +4,6 @@ import ( "time" "github.com/moleculer-go/moleculer" - payloadPkg "github.com/moleculer-go/moleculer/payload" "github.com/moleculer-go/moleculer/serializer" "github.com/moleculer-go/moleculer/transit" @@ -12,11 +11,12 @@ import ( ) type TCPTransporter struct { - options TCPOptions - tcpReader *TcpReader - tcpWriter *TcpWriter - udpServer *UdpServer - registry moleculer.Registry + options TCPOptions + tcpReader *TcpReader + tcpWriter *TcpWriter + udpServer *UdpServer + registry moleculer.Registry + gossipTimer *time.Ticker logger *log.Entry @@ -82,7 +82,7 @@ func (transporter *TCPTransporter) Connect(registry moleculer.Registry) chan err go func() { transporter.startTcpServer() transporter.startUDPServer() - transporter.startGossip() + transporter.startGossipTimer() endChan <- nil }() return endChan @@ -114,234 +114,6 @@ func (transporter *TCPTransporter) onTcpMessage(fromAddrss string, msgType int, } } -func (transporter *TCPTransporter) onGossipHello(fromAddrss string, msgBytes *[]byte) { - packet := transporter.serializer.BytesToPayload(msgBytes) - payload := packet.Get("payload") - nodeID := payload.Get("sender").String() - - node := transporter.registry.GetNodeByID(nodeID) - if node == nil { - // Unknown node. Register as offline node - node = transporter.registry.AddOfflineNode(nodeID, payload.Get("host").String(), payload.Get("port").Int()) - } - if node.GetUdpAddress() == "" { - node.UpdateInfo(nodeID, map[string]interface{}{ - "udpAddress": fromAddrss, - }) - } -} - -func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { - packet := transporter.serializer.BytesToPayload(msgBytes) - payload := packet.Get("payload") - - onlineResponse := map[string]interface{}{} - offlineResponse := map[string]interface{}{} - - transporter.registry.ForEachNode(func(node moleculer.Node) bool { - - onlineMap := payload.Get("online") - offlineMap := payload.Get("offline") - var seq int64 = 0 - var cpuSeq int64 = 0 - var cpu int64 = 0 - var offline moleculer.Payload - var online moleculer.Payload - - if offlineMap.Exists() { - offline = offlineMap.Get(node.GetID()) - if offline.Exists() { - transporter.logger.Debug("received seq for " + node.GetID()) - seq = offline.Int64() - } - } - if onlineMap.Exists() { - online = onlineMap.Get(node.GetID()) - if online.Exists() { - transporter.logger.Debug("received seq, cpuSeq, cpu for " + node.GetID()) - seq = online.Get("seq").Int64() - cpuSeq = online.Get("cpuSeq").Int64() - cpu = online.Get("cpu").Int64() - } - } - - if seq != 0 && seq < node.GetSequence() { - transporter.logger.Debug("We have newer info or requester doesn't know it") - if node.IsAvailable() { - info := node.ExportAsMap() - onlineResponse[node.GetID()] = []interface{}{info, node.GetCpuSequence(), node.GetCpu()} - transporter.logger.Debug("Node is available - send back the node info and cpu, cpuSed to " + node.GetID()) - } else { - offlineResponse[node.GetID()] = node.GetSequence() - transporter.logger.Debug("Node is offline - send back the seq to " + node.GetID()) - } - return true - } - - if offline != nil && offline.Exists() { - transporter.logger.Debug("Requester said it is OFFLINE") - if !node.IsAvailable() { - transporter.logger.Debug("We also know it as offline - update the seq") - if seq > node.GetSequence() { - node.UpdateInfo(node.GetID(), map[string]interface{}{ - "seq": seq, - }) - } - return true - } - - if !node.IsLocal() { - transporter.logger.Debug("our current state for it is online - change it to offline and update seq - nodeID:", node.GetID(), "seq:", seq) - // We know it is online, so we change it to offline - transporter.registry.DisconnectNode(node.GetID()) - - // Update the 'seq' to the received value - node.UpdateInfo(node.GetID(), map[string]interface{}{ - "seq": seq, - }) - return true - } - - if node.IsLocal() { - transporter.logger.Debug("msg is about the Local node - update the seq and send back info, cpu and cpuSeq") - // Update the 'seq' to the received value - node.UpdateInfo(node.GetID(), map[string]interface{}{ - "seq": seq + 1, - }) - onlineResponse[node.GetID()] = []interface{}{node.ExportAsMap(), node.GetCpuSequence(), node.GetCpu()} - return true - } - } - - if online != nil && online.Exists() { - // Requester said it is ONLINE - if node.IsAvailable() { - if cpuSeq > node.GetCpuSequence() { - // We update CPU info - node.UpdateInfo(node.GetID(), map[string]interface{}{ - "cpu": cpu, - "cpuSeq": cpuSeq, - }) - transporter.logger.Debug("CPU info updated for " + node.GetID()) - } else if cpuSeq < node.GetCpuSequence() { - // We have newer info, send back - //TODO check where we process this to see if we handle this array correctly - onlineResponse[node.GetID()] = []interface{}{node.GetCpuSequence(), node.GetCpu()} - transporter.logger.Debug("CPU info sent back to " + node.GetID()) - } - } else { - // We know it as offline. We do nothing, because we'll request it and we'll receive its INFO. - return true - } - } - return true - }) - - if len(onlineResponse) > 0 || len(offlineResponse) > 0 { - sender := payload.Get("sender").String() - transporter.Publish(msgTypeToCommand(PACKET_GOSSIP_RES), sender, payloadPkg.Empty().Add("online", onlineResponse).Add("offline", offlineResponse)) - transporter.logger.Debug("Gossip response sent to " + sender) - } else { - transporter.logger.Debug("No response sent to " + payload.Get("sender").String()) - } - -} - -func (transporter *TCPTransporter) onGossipResponse(msgBytes *[]byte) { - packet := transporter.serializer.BytesToPayload(msgBytes) - payload := packet.Get("payload") - sender := payload.Get("sender").String() - - online := payload.Get("online") - offline := payload.Get("offline") - - if online.Exists() { - transporter.logger.Debug("Received online info from nodeID: " + sender) - online.ForEach(func(key interface{}, value moleculer.Payload) bool { - nodeID, ok := key.(string) - if !ok { - transporter.logger.Error("Error parsing online nodeID") - return true - } - node := transporter.registry.GetNodeByID(nodeID) - if node != nil && node.IsLocal() { - transporter.logger.Debug("Received info about the local node - ignore it") - return true - } - row := online.Get(nodeID).Array() - info, cpu, cpuSeq := parseGossipResponse(row) - - if info != nil && (node != nil || node.GetSequence() < info["seq"].(int64)) { - transporter.logger.Debug("If we don't know it, or know, but has smaller seq, update 'info'") - info["sender"] = sender - transporter.registry.RemoteNodeInfoReceived(payloadPkg.New(info)) - } - if node != nil && cpuSeq > node.GetCpuSequence() { - transporter.logger.Debug("If we know it and has smaller cpuSeq, update 'cpu'") - node.HeartBeat(map[string]interface{}{ - "cpu": cpu, - "cpuSeq": cpuSeq, - }) - } - return true - }) - } - - if offline.Exists() { - transporter.logger.Debug("Received offline info from nodeID: " + sender) - offline.ForEach(func(key interface{}, value moleculer.Payload) bool { - nodeID, ok := key.(string) - if !ok { - transporter.logger.Error("Error parsing offline nodeID") - return true - } - node := transporter.registry.GetNodeByID(nodeID) - if node != nil && node.IsLocal() { - transporter.logger.Debug("Received info about the local node - ignore it") - return true - } - if node == nil { - return true - } - - seq := offline.Get(nodeID).Int64() - - if node.GetSequence() < seq { - if node.IsAvailable() { - transporter.logger.Debug("Node is online, will change it to offline") - transporter.registry.DisconnectNode(nodeID) - } - node.UpdateInfo(nodeID, map[string]interface{}{ - "seq": seq, - }) - } - return true - }) - } -} - -func parseGossipResponse(row []moleculer.Payload) (info map[string]interface{}, cpu int64, cpuSeq int64) { - cpuSeq = -1 - cpu = -1 - if len(row) == 1 { - info = row[0].RawMap() - } - if len(row) == 2 { - cpuSeq = row[0].Int64() - cpu = row[1].Int64() - } - if len(row) == 3 { - info = row[0].RawMap() - cpuSeq = row[1].Int64() - cpu = row[2].Int64() - } - return info, cpu, cpuSeq -} - -func isGossipMessage(msgType byte) bool { - return msgType == PACKET_GOSSIP_REQ || msgType == PACKET_GOSSIP_RES || msgType == PACKET_GOSSIP_HELLO -} - func msgTypeToCommand(msgType int) string { switch msgType { case PACKET_EVENT: @@ -372,9 +144,35 @@ func msgTypeToCommand(msgType int) string { return "???" } } +func commandToMsgType(command string) int { + switch command { + case "EVENT": + return PACKET_EVENT + case "REQ": + return PACKET_REQUEST + case "RES": + return PACKET_RESPONSE + case "PING": + return PACKET_PING + case "PONG": + return PACKET_PONG + case "GOSSIP_REQ": + return PACKET_GOSSIP_REQ + case "GOSSIP_RES": + return PACKET_GOSSIP_RES + case "GOSSIP_HELLO": + return PACKET_GOSSIP_HELLO + default: + return -1 + } +} func (transporter *TCPTransporter) incomingMessage(msgType int, msgBytes *[]byte) { command := msgTypeToCommand(msgType) + if command == "???" { + transporter.logger.Error("Unknown command received - msgType: " + string(msgType)) + return + } message := transporter.serializer.BytesToPayload(msgBytes) if transporter.validateMsg(message) { if handlers, ok := transporter.handlers[command]; ok { @@ -411,7 +209,7 @@ func (transporter *TCPTransporter) startUDPServer() { err := transporter.udpServer.Start() if err != nil { - transporter.logger.Error("Error starting UDP server:", err) + transporter.logger.Error("TCPTransporter.startUDPServer() Error starting UDP server:", err) } } @@ -456,20 +254,26 @@ func (transporter *TCPTransporter) onUdpMessage(nodeID, address string, port int } } -func (transporter *TCPTransporter) startGossip() { - transporter.gossip = &Gossip{} -} - func (transporter *TCPTransporter) Disconnect() chan error { endChan := make(chan error) go func() { - // Additional disconnection logic goes here + transporter.tcpReader.Close() + transporter.tcpWriter.Close() + transporter.udpServer.Close() + if transporter.gossipTimer != nil { + transporter.gossipTimer.Stop() + } endChan <- nil }() return endChan } func (transporter *TCPTransporter) Subscribe(command, nodeID string, handler transit.TransportHandler) { + if commandToMsgType(command) == -1 { + transporter.logger.Error("TCPTransporter.Subscribe() Invalid command: " + command) + return + } + if _, ok := transporter.handlers[command]; !ok { transporter.handlers[command] = make([]transit.TransportHandler, 0) } @@ -477,7 +281,16 @@ func (transporter *TCPTransporter) Subscribe(command, nodeID string, handler tra } func (transporter *TCPTransporter) Publish(command, nodeID string, message moleculer.Payload) { - // Additional publish logic goes here + msgType := commandToMsgType(command) + if msgType == -1 { + transporter.logger.Error("TCPTransporter.Publish() Invalid command: " + command) + return + } + msgBts := transporter.serializer.PayloadToBytes(message) + err := transporter.tcpWriter.Send(nodeID, byte(msgType), msgBts) + if err != nil { + transporter.logger.Error("TCPTransporter.Publish() Error sending message: ", err) + } } func (transporter *TCPTransporter) SetPrefix(prefix string) { diff --git a/transit/tcp/tcp-writer.go b/transit/tcp/tcp-writer.go index 0fef29a..2f831a2 100644 --- a/transit/tcp/tcp-writer.go +++ b/transit/tcp/tcp-writer.go @@ -68,7 +68,7 @@ func (w *TcpWriter) Connect(nodeID, host string, port int) (*net.TCPConn, error) func (w *TcpWriter) Send(nodeID string, msgType byte, msgBytes []byte) error { socket, exists := w.sockets[nodeID] if !exists || socket == nil { - return errors.New("connection does not exist") + return errors.New("connection does not exist for nodeID: " + nodeID) } header := make([]byte, HEADER_SIZE) binary.BigEndian.PutUint32(header[1:], uint32(len(msgBytes)+len(header))) diff --git a/transit/tcp/udp.go b/transit/tcp/udp.go index abb34ff..6b08137 100644 --- a/transit/tcp/udp.go +++ b/transit/tcp/udp.go @@ -20,7 +20,7 @@ type UdpServerEntry struct { type OnUdpMessage func(nodeID, ip string, port int) type UdpServer struct { - state string + state State opts UdpServerOptions discoveryCounter int logger *log.Entry @@ -205,7 +205,7 @@ func (u *UdpServer) Start() error { return u.startServer(u.opts.BindAddress, u.opts.Port, "", 0, broadcastAddrss) } - u.state = "started" + u.state = STARTED go u.firstDiscoveryMessage() @@ -257,7 +257,7 @@ func (u *UdpServer) firstDiscoveryMessage() { func (u *UdpServer) handleIncomingMessagesForServer(server *UdpServerEntry) { buffer := make([]byte, 2048) - for u.state == "started" { + for u.state == STARTED { n, addr, err := server.conn.ReadFromUDP(buffer) if err != nil { u.logger.Errorln("Error reading from UDP:", err) @@ -297,6 +297,10 @@ func (u *UdpServer) startDiscovering() { u.logger.Info("UDP Discovery is disabled.") return } + if u.discoverTimer != nil { + u.logger.Warn("Discovery already started.") + return + } u.discoverTimer = time.NewTicker(u.opts.DiscoverPeriod) go func() { for range u.discoverTimer.C { @@ -335,12 +339,11 @@ func (u *UdpServer) StopDiscovering() { u.discoverTimer.Stop() u.discoverTimer = nil } - u.state = "stopped" } func (u *UdpServer) Close() { u.logger.Info("Closing UDP Server.") - u.state = "stopped" + u.state = STOPPED u.StopDiscovering() for _, server := range u.servers { if server.conn != nil { diff --git a/util/metrics.go b/util/metrics.go new file mode 100644 index 0000000..d6269be --- /dev/null +++ b/util/metrics.go @@ -0,0 +1,50 @@ +package util + +import ( + "time" + + "github.com/shirou/gopsutil/cpu" +) + +func GetCpuUsage(sampleTime time.Duration) (float64, error) { + first, err := cpu.Times(false) + if err != nil { + return 0, err + } + + time.Sleep(sampleTime) + + second, err := cpu.Times(false) + if err != nil { + return 0, err + } + + time.Sleep(sampleTime) + + third, err := cpu.Times(false) + if err != nil { + return 0, err + } + + var totalPerc float64 + for i, _ := range first { + firstUsage := calcUsage(first[i], second[i]) + secondUsage := calcUsage(second[i], third[i]) + totalPerc += (firstUsage + secondUsage) / 2 + } + + avg := totalPerc / float64(len(first)) + return avg, nil +} + +func calcUsage(first cpu.TimesStat, second cpu.TimesStat) float64 { + firstTotal := getTotalTime(first) + secondTotal := getTotalTime(second) + total := secondTotal - firstTotal + idle := second.Idle - first.Idle + return (total - idle) / total +} + +func getTotalTime(t cpu.TimesStat) float64 { + return t.User + t.System + t.Idle + t.Nice + t.Iowait + t.Irq + t.Softirq + t.Steal +} From ba550162e473511c19fc345bb5539e348e6c5e67 Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Thu, 11 Apr 2024 15:49:14 +1200 Subject: [PATCH 11/20] update connect method --- transit/amqp/amqp.go | 7 ++++--- transit/kafka/kafka.go | 2 +- transit/pubsub/pubsub.go | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/transit/amqp/amqp.go b/transit/amqp/amqp.go index 586b698..7ea7365 100644 --- a/transit/amqp/amqp.go +++ b/transit/amqp/amqp.go @@ -2,14 +2,15 @@ package amqp import ( "fmt" + "strings" + "time" + "github.com/moleculer-go/moleculer" "github.com/moleculer-go/moleculer/serializer" "github.com/moleculer-go/moleculer/transit" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/streadway/amqp" - "strings" - "time" ) const ( @@ -129,7 +130,7 @@ func CreateAmqpTransporter(options AmqpOptions) transit.Transport { } } -func (t *AmqpTransporter) Connect() chan error { +func (t *AmqpTransporter) Connect(registry moleculer.Registry) chan error { endChan := make(chan error) go func() { diff --git a/transit/kafka/kafka.go b/transit/kafka/kafka.go index e1ee9be..9653159 100644 --- a/transit/kafka/kafka.go +++ b/transit/kafka/kafka.go @@ -89,7 +89,7 @@ func CreateKafkaTransporter(options KafkaOptions) transit.Transport { } } -func (t *KafkaTransporter) Connect() chan error { +func (t *KafkaTransporter) Connect(registry moleculer.Registry) chan error { endChan := make(chan error) go func() { t.logger.Debug("Kafka Connect() - url: ", t.opts.Url) diff --git a/transit/pubsub/pubsub.go b/transit/pubsub/pubsub.go index 689daf5..9540751 100644 --- a/transit/pubsub/pubsub.go +++ b/transit/pubsub/pubsub.go @@ -164,6 +164,7 @@ func isKafka(v string) bool { // CreateTransport : based on config it will load the transporter // for now is hard coded for NATS Streaming localhost func (pubsub *PubSub) createTransport() transit.Transport { + pubsub.logger.Debug("createTransport() Config.Transporter: " + pubsub.broker.Config.Transporter) var transport transit.Transport if pubsub.broker.Config.TransporterFactory != nil { pubsub.logger.Info("Transporter: Custom factory") From 4577208d1f7c604fdf3797d875b2825d89069fbe Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Thu, 11 Apr 2024 15:54:13 +1200 Subject: [PATCH 12/20] fix compilation issues --- registry/node.go | 2 +- transit/memory/memory.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/registry/node.go b/registry/node.go index b236398..d46cd58 100644 --- a/registry/node.go +++ b/registry/node.go @@ -171,7 +171,7 @@ func (node *Node) Update(id string, info map[string]interface{}) (bool, []map[st } func (node *Node) UpdateMetrics() { - cpu, err := util.GetCpuUsage() + cpu, err := util.GetCpuUsage(100 * time.Millisecond) if err != nil { node.logger.Error("Error getting cpu usage:", err) return diff --git a/transit/memory/memory.go b/transit/memory/memory.go index df6a270..8b68bc8 100644 --- a/transit/memory/memory.go +++ b/transit/memory/memory.go @@ -52,7 +52,7 @@ func (transporter *MemoryTransporter) SetSerializer(serializer serializer.Serial } -func (transporter *MemoryTransporter) Connect() chan error { +func (transporter *MemoryTransporter) Connect(registry moleculer.Registry) chan error { transporter.logger.Debug("[Mem-Trans-", transporter.instanceID, "] -> Connecting() ...") endChan := make(chan error) go func() { From 225ef19bf5b44e827891b36f69f428a55af2cd07 Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Thu, 11 Apr 2024 18:02:02 +1200 Subject: [PATCH 13/20] connect to nodes --- moleculer.go | 2 + registry/node.go | 16 +++++-- transit/pubsub/pubsub.go | 1 + transit/tcp/tcp-transporter.go | 73 +++++++++++++++++++++++++++---- transit/tcp/tcp-writer.go | 34 +++++++++++++++ transit/tcp/udp.go | 80 ++++++++++------------------------ 6 files changed, 137 insertions(+), 69 deletions(-) diff --git a/moleculer.go b/moleculer.go index f357916..e26c531 100644 --- a/moleculer.go +++ b/moleculer.go @@ -226,6 +226,7 @@ type Node interface { ExportAsMap() map[string]interface{} IsAvailable() bool GetIpList() []string + GetPort() int Available() Unavailable() IsExpired(timeout time.Duration) bool @@ -241,6 +242,7 @@ type Node interface { IsLocal() bool Disconnected(isUnexpected bool) UpdateMetrics() + GetHostname() string } type Options struct { diff --git a/registry/node.go b/registry/node.go index d46cd58..b1c87ca 100644 --- a/registry/node.go +++ b/registry/node.go @@ -89,6 +89,18 @@ func (node *Node) GetIpList() []string { return node.ipList } +func (node *Node) GetUdpAddress() string { + return node.udpAddress +} + +func (node *Node) GetHostname() string { + return node.hostname +} + +func (node *Node) GetPort() int { + return node.port +} + func (node *Node) UpdateInfo(id string, info map[string]interface{}) []map[string]interface{} { if id != node.id { node.logger.Error(fmt.Sprintf("Node.Update() - the id received : %s does not match this node.id : %s", id, node.id)) @@ -152,10 +164,6 @@ func (node *Node) IsLocal() bool { return node.isLocal } -func (node *Node) GetUdpAddress() string { - return node.udpAddress -} - func (node *Node) Update(id string, info map[string]interface{}) (bool, []map[string]interface{}) { if id != node.id { node.logger.Error(fmt.Sprintf("Node.Update() - the id received : %s does not match this node.id : %s", id, node.id)) diff --git a/transit/pubsub/pubsub.go b/transit/pubsub/pubsub.go index 9540751..38ac261 100644 --- a/transit/pubsub/pubsub.go +++ b/transit/pubsub/pubsub.go @@ -266,6 +266,7 @@ func (pubsub *PubSub) createTCPTransporter() transit.Transport { // Maximum TCP packet size MaxPacketSize: 1 * 1024 * 1024, + Namespace: pubsub.broker.Config.Namespace, NodeId: pubsub.broker.LocalNode().GetID(), Logger: pubsub.logger.WithField("transport", "tcp"), Serializer: pubsub.serializer, diff --git a/transit/tcp/tcp-transporter.go b/transit/tcp/tcp-transporter.go index e64a94e..f9281fe 100644 --- a/transit/tcp/tcp-transporter.go +++ b/transit/tcp/tcp-transporter.go @@ -1,6 +1,8 @@ package tcp import ( + "errors" + "strconv" "time" "github.com/moleculer-go/moleculer" @@ -65,6 +67,7 @@ type TCPOptions struct { Prefix string NodeId string + Namespace string Logger *log.Entry Serializer serializer.Serializer ValidateMsg transit.ValidateMsgFunc @@ -72,6 +75,9 @@ type TCPOptions struct { func CreateTCPTransporter(options TCPOptions) TCPTransporter { transport := TCPTransporter{options: options, logger: options.Logger} + transport.handlers = make(map[string][]transit.TransportHandler) + transport.serializer = options.Serializer + transport.validateMsg = options.ValidateMsg return transport } @@ -187,6 +193,7 @@ func (transporter *TCPTransporter) startTcpServer() { transporter.tcpReader = NewTcpReader(transporter.options.Port, transporter.onTcpMessage, transporter.logger.WithFields(log.Fields{ "TCPTransporter": "TCPReader", })) + transporter.tcpReader.Listen() transporter.tcpWriter = NewTcpWriter(transporter.options.MaxConnections, transporter.logger.WithFields(log.Fields{ "TCPTransporter": "TCPWriter", })) @@ -202,7 +209,8 @@ func (transporter *TCPTransporter) startUDPServer() { DiscoverPeriod: transporter.options.UdpPeriod, MaxDiscovery: transporter.options.UdpMaxDiscovery, Discovery: transporter.options.UdpDiscovery, - // Namespace: transporter.options.Namespace, TODO + NodeID: transporter.options.NodeId, + Namespace: transporter.options.Namespace, }, transporter.onUdpMessage, transporter.logger.WithFields(log.Fields{ "TCPTransporter": "UdpServer", })) @@ -269,27 +277,76 @@ func (transporter *TCPTransporter) Disconnect() chan error { } func (transporter *TCPTransporter) Subscribe(command, nodeID string, handler transit.TransportHandler) { - if commandToMsgType(command) == -1 { - transporter.logger.Error("TCPTransporter.Subscribe() Invalid command: " + command) - return - } - + // if commandToMsgType(command) == -1 { + // transporter.logger.Error("TCPTransporter.Subscribe() Invalid command: " + command) + // return + // } if _, ok := transporter.handlers[command]; !ok { transporter.handlers[command] = make([]transit.TransportHandler, 0) } transporter.handlers[command] = append(transporter.handlers[command], handler) } +func (transporter *TCPTransporter) getNodeAddress(node moleculer.Node) string { + if node.GetUdpAddress() != "" { + return node.GetUdpAddress() + } + if transporter.options.UseHostname && node.GetHostname() != "" { + return node.GetHostname() + } + if len(node.GetIpList()) > 0 { + return node.GetIpList()[0] + } + return "" +} + +func (transporter *TCPTransporter) tryToConnect(nodeID string) error { + node := transporter.registry.GetNodeByID(nodeID) + if node == nil { + transporter.logger.Error("TCPTransporter.tryToConnect() Unknown nodeID: " + nodeID) + return errors.New("Unknown nodeID: " + nodeID) + } + nodeAddress := transporter.getNodeAddress(node) + if nodeAddress == "" { + transporter.logger.Error("TCPTransporter.tryToConnect() No address found for nodeID: " + nodeID) + return errors.New("No address found for nodeID: " + nodeID) + } + _, err := transporter.tcpWriter.Connect(nodeID, nodeAddress, node.GetPort()) + if err != nil { + transporter.logger.Error("TCPTransporter.tryToConnect() Error connecting to nodeID: "+nodeID+" node address:"+nodeAddress+" port: "+strconv.Itoa(node.GetPort())+" error: ", err) + return err + } + transporter.logger.Info("TCPTransporter.tryToConnect() Connected to nodeID: " + nodeID + " node address:" + nodeAddress + " port: " + strconv.Itoa(node.GetPort())) + return nil +} + func (transporter *TCPTransporter) Publish(command, nodeID string, message moleculer.Payload) { msgType := commandToMsgType(command) if msgType == -1 { - transporter.logger.Error("TCPTransporter.Publish() Invalid command: " + command) + transporter.logger.Error("TCPTransporter.Publish() Invalid command: " + command + " nodeID: " + nodeID) return } + msgBts := transporter.serializer.PayloadToBytes(message) + + if nodeID == "" { + err := transporter.tcpWriter.Broadcast(byte(msgType), msgBts) + if err != nil { + transporter.logger.Error("TCPTransporter.Publish() Error broadcasting message command:"+command+" error: ", err) + } + return + } + + if !transporter.tcpWriter.IsConnected(nodeID) { + err := transporter.tryToConnect(nodeID) + if err != nil { + transporter.logger.Error("TCPTransporter.Publish() Error connecting to nodeID: "+nodeID+" error: ", err) + return + } + } err := transporter.tcpWriter.Send(nodeID, byte(msgType), msgBts) if err != nil { - transporter.logger.Error("TCPTransporter.Publish() Error sending message: ", err) + transporter.logger.Error("TCPTransporter.Publish() Error sending message command:"+command+" error: ", err) } } diff --git a/transit/tcp/tcp-writer.go b/transit/tcp/tcp-writer.go index 2f831a2..014b565 100644 --- a/transit/tcp/tcp-writer.go +++ b/transit/tcp/tcp-writer.go @@ -65,8 +65,42 @@ func (w *TcpWriter) Connect(nodeID, host string, port int) (*net.TCPConn, error) return conn, nil } +func (w *TcpWriter) IsConnected(nodeID string) bool { + w.lock.Lock() + defer w.lock.Unlock() + _, exists := w.sockets[nodeID] + return exists +} + +func (w *TcpWriter) Broadcast(msgType byte, msgBytes []byte) error { + w.lock.Lock() + nodeIDs := make([]string, 0, len(w.sockets)) + for nodeID, _ := range w.sockets { + nodeIDs = append(nodeIDs, nodeID) + } + w.lock.Unlock() + + var lastError error + errorCount := 0 + for _, nodeID := range nodeIDs { + err := w.Send(nodeID, msgType, msgBytes) + if err != nil { + w.logger.Errorf("Error sending message to node %s: %s", nodeID, err) + lastError = err + errorCount++ + } + } + if errorCount > 0 { + w.logger.Errorf("Failed to send message to %d nodes last error: %s", errorCount, lastError) + return errors.New("Failed to send message to " + fmt.Sprint(errorCount) + " nodes last error: " + lastError.Error()) + } + return nil +} + func (w *TcpWriter) Send(nodeID string, msgType byte, msgBytes []byte) error { + w.lock.Lock() socket, exists := w.sockets[nodeID] + w.lock.Unlock() if !exists || socket == nil { return errors.New("connection does not exist for nodeID: " + nodeID) } diff --git a/transit/tcp/udp.go b/transit/tcp/udp.go index 6b08137..7498ca2 100644 --- a/transit/tcp/udp.go +++ b/transit/tcp/udp.go @@ -14,6 +14,7 @@ import ( type UdpServerEntry struct { conn *net.UDPConn + packetConn *ipv4.PacketConn discoveryTargets []string } @@ -62,10 +63,10 @@ func (u *UdpServer) startServer(ip string, port int, multicast string, multicast u.logger.Warnf("Unable to listen on UDP address: %s\n", err) return err } - + var packetConn *ipv4.PacketConn if multicast != "" { u.logger.Infof("UDP Multicast Server is listening on %s:%d. Membership: %s \n", ip, port, multicast) - err := u.joinMulticastGroup(multicast, udpConn, multicastTTL, ip, port) + packetConn, err = u.joinMulticastGroup(multicast, udpConn, multicastTTL, ip, port) if err != nil { u.logger.Error("Error joining multicast group:", err) return err @@ -75,74 +76,38 @@ func (u *UdpServer) startServer(ip string, port int, multicast string, multicast u.logger.Infof("UDP Broadcast Server is listening on %s:%d\n", ip, port) } - // // Start a goroutine to handle incoming messages - // go func() { - // buf := make([]byte, 1024) - // for { - // n, addr, err := udpConn.ReadFromUDP(buf) - // if err != nil { - // u.logger.Warnf("Error reading from UDP: %s\n", err) - // break - // } - - // // Handle the message - // u.onMessage(buf[:n], addr) - // } - // }() - // Add the connection to your list of servers - u.servers = append(u.servers, &UdpServerEntry{udpConn, discoveryTargets}) + u.servers = append(u.servers, &UdpServerEntry{udpConn, packetConn, discoveryTargets}) return nil } -// func (u *UdpServer) onMessage(buf []byte, addr *net.UDPAddr) { -// msg := string(buf) -// u.logger.Debugf("UDP message received from %s: %s\n", addr.String(), msg) - -// parts := strings.Split(msg, "|") -// if len(parts) != 3 { -// u.logger.Debugf("Malformed UDP packet received: %s\n", msg) -// return -// } - -// if parts[0] == u.namespace { -// port, err := strconv.Atoi(parts[2]) -// if err != nil { -// u.logger.Debugf("UDP packet process error: %s\n", err) -// return -// } - -// u.emit("message", parts[1], addr.IP.String(), port) -// } -// } - -func (u *UdpServer) joinMulticastGroup(multicast string, udpConn *net.UDPConn, multicastTTL int, ip string, port int) error { - groupAddr, err := net.ResolveUDPAddr("udp4", multicast) +func (u *UdpServer) joinMulticastGroup(multicast string, udpConn *net.UDPConn, multicastTTL int, ip string, port int) (*ipv4.PacketConn, error) { + groupAddr, err := net.ResolveUDPAddr("udp4", multicast+":"+strconv.Itoa(port)) if err != nil { u.logger.Warnf("Unable to resolve multicast address: %s\n", err) - return err + return nil, err } - p := ipv4.NewPacketConn(udpConn) + packetConn := ipv4.NewPacketConn(udpConn) interfaces, err := net.Interfaces() if err != nil { u.logger.Warnf("Unable to get network interfaces: %s\n", err) - return err + return nil, err } for _, iface := range interfaces { - if err := p.JoinGroup(&iface, groupAddr); err != nil { + if err := packetConn.JoinGroup(&iface, groupAddr); err != nil { u.logger.Warnf("Unable to join multicast group on interface %s: %s\n", iface.Name, err) } } - if err := p.SetMulticastTTL(multicastTTL); err != nil { + if err := packetConn.SetMulticastTTL(multicastTTL); err != nil { u.logger.Warnf("Unable to set multicast TTL: %s\n", err) - return err + return nil, err } u.logger.Infof("UDP Multicast Server is listening on %s:%d. Membership: %s\n", ip, port, multicast) - return nil + return packetConn, nil } func (u *UdpServer) getAllIPs() []string { @@ -169,11 +134,12 @@ func (u *UdpServer) getAllIPs() []string { ip = v.IP } - if ip == nil { + // Check if the IP is a valid IPv4 address + if ip == nil || ip.To4() == nil { continue } ips = append(ips, ip.String()) - u.logger.Debug("Interface: %v, IP Address: %v", i.Name, ip.String()) + u.logger.Debugf("Interface: %v, IP Address: %v", i.Name, ip.String()) } } return ips @@ -264,26 +230,26 @@ func (u *UdpServer) handleIncomingMessagesForServer(server *UdpServerEntry) { continue } message := string(buffer[:n]) - u.logger.Debug("Received message from %s: %s", addr.String(), message) + u.logger.Debugf("Received message from %s: %s", addr.String(), message) // Parse the message parts := strings.Split(message, "|") if len(parts) != 3 { - u.logger.Debug("Malformed UDP packet received: %s", message) + u.logger.Debugf("Malformed UDP packet received: %s", message) continue } namespace := parts[0] if namespace != u.opts.Namespace { - u.logger.Debug("Message received for a different namespace: %s", namespace) + u.logger.Debugf("Message received for a different namespace: %s", namespace) continue } nodeID := parts[1] port, err := strconv.Atoi(parts[2]) if err != nil { - u.logger.Debug("Error parsing port number: %s", err) + u.logger.Debugf("Error parsing port number: %s", err) continue } @@ -293,7 +259,7 @@ func (u *UdpServer) handleIncomingMessagesForServer(server *UdpServerEntry) { } func (u *UdpServer) startDiscovering() { - if u.opts.Discovery == false { + if !u.opts.Discovery { u.logger.Info("UDP Discovery is disabled.") return } @@ -301,7 +267,7 @@ func (u *UdpServer) startDiscovering() { u.logger.Warn("Discovery already started.") return } - u.discoverTimer = time.NewTicker(u.opts.DiscoverPeriod) + u.discoverTimer = time.NewTicker(u.opts.DiscoverPeriod * time.Second) go func() { for range u.discoverTimer.C { u.broadcastDiscoveryMessage() @@ -319,7 +285,7 @@ func (u *UdpServer) broadcastDiscoveryMessage() { u.discoveryCounter++ for _, server := range u.servers { for _, target := range server.discoveryTargets { - destAddr, err := net.ResolveUDPAddr("udp4", target+":"+strconv.Itoa(u.opts.Port)) + destAddr, err := net.ResolveUDPAddr("udp4", fmt.Sprintf("%s:%d", target, u.opts.Port)) if err != nil { u.logger.Error("Error resolving UDP address:", err) continue From d33b165748d1d589c7a24a50d609852a0439dd1c Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Thu, 11 Apr 2024 20:47:33 +1200 Subject: [PATCH 14/20] update port after listener starts --- moleculer.go | 2 +- registry/node.go | 8 ++------ registry/registry.go | 2 +- transit/tcp/gossip.go | 12 +++++------ transit/tcp/tcp-reader.go | 32 +++++++++++++++++++---------- transit/tcp/tcp-transporter.go | 37 ++++++++++++++++++++++++++++------ transit/tcp/udp.go | 18 ++++++++++------- 7 files changed, 73 insertions(+), 38 deletions(-) diff --git a/moleculer.go b/moleculer.go index e26c531..40bf076 100644 --- a/moleculer.go +++ b/moleculer.go @@ -231,7 +231,7 @@ type Node interface { Unavailable() IsExpired(timeout time.Duration) bool Update(id string, info map[string]interface{}) (bool, []map[string]interface{}) - UpdateInfo(id string, info map[string]interface{}) []map[string]interface{} + UpdateInfo(info map[string]interface{}) []map[string]interface{} IncreaseSequence() HeartBeat(heartbeat map[string]interface{}) Publish(service map[string]interface{}) diff --git a/registry/node.go b/registry/node.go index b1c87ca..d6d2ce3 100644 --- a/registry/node.go +++ b/registry/node.go @@ -101,11 +101,7 @@ func (node *Node) GetPort() int { return node.port } -func (node *Node) UpdateInfo(id string, info map[string]interface{}) []map[string]interface{} { - if id != node.id { - node.logger.Error(fmt.Sprintf("Node.Update() - the id received : %s does not match this node.id : %s", id, node.id)) - return []map[string]interface{}{} - } +func (node *Node) UpdateInfo(info map[string]interface{}) []map[string]interface{} { node.logger.Debug("node.UpdateInfo() - info:") node.logger.Debug(util.PrettyPrintMap(info)) @@ -170,7 +166,7 @@ func (node *Node) Update(id string, info map[string]interface{}) (bool, []map[st return false, nil } node.logger.Debug("node.Update()") - removedServices := node.UpdateInfo(id, info) + removedServices := node.UpdateInfo(info) reconnected := !node.isAvailable node.isAvailable = true node.lastHeartBeatTime = time.Now().Unix() diff --git a/registry/registry.go b/registry/registry.go index 15bda07..9473c72 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -118,7 +118,7 @@ func (registry *ServiceRegistry) LocalNode() moleculer.Node { // Register a node as offline because we don't know all information about it func (registry *ServiceRegistry) AddOfflineNode(nodeID string, address string, port int) moleculer.Node { node := CreateNode(nodeID, false, registry.logger.WithField("Node", nodeID)) - node.UpdateInfo(nodeID, map[string]interface{}{ + node.UpdateInfo(map[string]interface{}{ "hostname": address, "port": port, "ipList": []string{address}, diff --git a/transit/tcp/gossip.go b/transit/tcp/gossip.go index 7ed38bd..2b1de51 100644 --- a/transit/tcp/gossip.go +++ b/transit/tcp/gossip.go @@ -84,7 +84,7 @@ func (transporter *TCPTransporter) onGossipHello(fromAddrss string, msgBytes *[] node = transporter.registry.AddOfflineNode(sender, payload.Get("host").String(), payload.Get("port").Int()) } if node.GetUdpAddress() == "" { - node.UpdateInfo(sender, map[string]interface{}{ + node.UpdateInfo(map[string]interface{}{ "udpAddress": fromAddrss, }) } @@ -145,7 +145,7 @@ func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { if !node.IsAvailable() { transporter.logger.Debug("We also know it as offline - update the seq") if seq > node.GetSequence() { - node.UpdateInfo(node.GetID(), map[string]interface{}{ + node.UpdateInfo(map[string]interface{}{ "seq": seq, }) } @@ -158,7 +158,7 @@ func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { transporter.registry.DisconnectNode(node.GetID()) // Update the 'seq' to the received value - node.UpdateInfo(node.GetID(), map[string]interface{}{ + node.UpdateInfo(map[string]interface{}{ "seq": seq, }) return true @@ -167,7 +167,7 @@ func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { if node.IsLocal() { transporter.logger.Debug("msg is about the Local node - update the seq and send back info, cpu and cpuSeq") // Update the 'seq' to the received value - node.UpdateInfo(node.GetID(), map[string]interface{}{ + node.UpdateInfo(map[string]interface{}{ "seq": seq + 1, }) onlineResponse[node.GetID()] = []interface{}{node.ExportAsMap(), node.GetCpuSequence(), node.GetCpu()} @@ -180,7 +180,7 @@ func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { if node.IsAvailable() { if cpuSeq > node.GetCpuSequence() { // We update CPU info - node.UpdateInfo(node.GetID(), map[string]interface{}{ + node.UpdateInfo(map[string]interface{}{ "cpu": cpu, "cpuSeq": cpuSeq, }) @@ -274,7 +274,7 @@ func (transporter *TCPTransporter) onGossipResponse(msgBytes *[]byte) { transporter.logger.Debug("Node is online, will change it to offline") transporter.registry.DisconnectNode(nodeID) } - node.UpdateInfo(nodeID, map[string]interface{}{ + node.UpdateInfo(map[string]interface{}{ "seq": seq, }) } diff --git a/transit/tcp/tcp-reader.go b/transit/tcp/tcp-reader.go index 9dc66b6..47d5778 100644 --- a/transit/tcp/tcp-reader.go +++ b/transit/tcp/tcp-reader.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "fmt" "net" + "strconv" "sync" log "github.com/sirupsen/logrus" @@ -38,11 +39,26 @@ func NewTcpReader(port int, onMessage OnMessageFunc, logger *log.Entry) *TcpRead } } -func (r *TcpReader) Listen() { +func (r *TcpReader) Listen() (int, error) { var err error r.listener, err = net.Listen("tcp", fmt.Sprintf(":%d", r.port)) if err != nil { - r.logger.Fatal("Server error: ", err) + r.logger.Error("Server error: ", err) + return 0, err + } + + if r.port == 0 { + _, portString, err := net.SplitHostPort(r.listener.Addr().String()) + if err != nil { + r.logger.Error("Could not net.SplitHostPort() error: ", err) + return 0, err + } + port, err := strconv.Atoi(portString) + if err != nil { + r.logger.Error("Could not convert port to integer error: ", err) + return 0, err + } + r.port = port } r.logger.Infof("TCP server is listening on port %d", r.port) @@ -62,30 +78,28 @@ func (r *TcpReader) Listen() { go r.handleConnection(conn) } }() + return r.port, nil } func (r *TcpReader) handleConnection(conn net.Conn) { address := conn.RemoteAddr().String() r.logger.Debugf("New TCP client connected from '%s'\n", address) - var err error - for err == nil { msgType, msgBytes, e := r.readMessage(conn) err = e - if err == nil { + if err != nil { r.logger.Errorf("Error reading message from '%s': %s", address, err) break } + r.logger.Debug("handleConnection() message read from socket - msgType: ", msgType, "message:", string(msgBytes)) r.onMessage(address, msgType, &msgBytes) } - r.closeSocket(conn) } func (r *TcpReader) readMessage(conn net.Conn) (msgType int, msg []byte, err error) { var buf []byte - for { // Read data from the connection chunk := make([]byte, 256) @@ -94,24 +108,20 @@ func (r *TcpReader) readMessage(conn net.Conn) (msgType int, msg []byte, err err return 0, nil, err } chunk = chunk[:n] - // If there's a previous chunk, concatenate them if buf != nil { buf = append(buf, chunk...) } else { buf = chunk } - // If the buffer is too short, wait for the next chunk if len(buf) < 6 { continue } - // If the buffer is larger than the max packet size, return an error if r.maxPacketSize > 0 && len(buf) > r.maxPacketSize { return 0, nil, fmt.Errorf("incoming packet is larger than the 'maxPacketSize' limit (%d > %d)", len(buf), r.maxPacketSize) } - // Check the CRC crc := buf[1] ^ buf[2] ^ buf[3] ^ buf[4] ^ buf[5] if crc != buf[0] { diff --git a/transit/tcp/tcp-transporter.go b/transit/tcp/tcp-transporter.go index f9281fe..d86f45c 100644 --- a/transit/tcp/tcp-transporter.go +++ b/transit/tcp/tcp-transporter.go @@ -51,7 +51,7 @@ type TCPOptions struct { // Send broadcast (Boolean, String, Array) UdpBroadcast []string UdpBroadcastAddrs []string - // TCP server port. Null or 0 means random port + // TCP server port. 0 means random port Port int // Static remote nodes address list (when UDP discovery is not available) Urls []string @@ -193,10 +193,19 @@ func (transporter *TCPTransporter) startTcpServer() { transporter.tcpReader = NewTcpReader(transporter.options.Port, transporter.onTcpMessage, transporter.logger.WithFields(log.Fields{ "TCPTransporter": "TCPReader", })) - transporter.tcpReader.Listen() transporter.tcpWriter = NewTcpWriter(transporter.options.MaxConnections, transporter.logger.WithFields(log.Fields{ "TCPTransporter": "TCPWriter", })) + + port, err := transporter.tcpReader.Listen() + if err != nil { + transporter.logger.Error("Error trying to listen on tcp reader - error: ", err) + return + } + node := transporter.registry.GetLocalNode() + node.UpdateInfo(map[string]interface{}{ + "port": port, + }) } func (transporter *TCPTransporter) startUDPServer() { @@ -211,7 +220,7 @@ func (transporter *TCPTransporter) startUDPServer() { Discovery: transporter.options.UdpDiscovery, NodeID: transporter.options.NodeId, Namespace: transporter.options.Namespace, - }, transporter.onUdpMessage, transporter.logger.WithFields(log.Fields{ + }, transporter.registry, transporter.onUdpMessage, transporter.logger.WithFields(log.Fields{ "TCPTransporter": "UdpServer", })) @@ -250,13 +259,13 @@ func (transporter *TCPTransporter) onUdpMessage(nodeID, address string, port int node = transporter.registry.AddOfflineNode(nodeID, address, port) } else if !node.IsAvailable() { ipList := addIpToList(node.GetIpList(), address) - node.UpdateInfo(nodeID, map[string]interface{}{ + node.UpdateInfo(map[string]interface{}{ "hostname": address, "port": port, "ipList": ipList, }) } - node.UpdateInfo(nodeID, map[string]interface{}{ + node.UpdateInfo(map[string]interface{}{ "udpAddress": address, }) } @@ -321,12 +330,28 @@ func (transporter *TCPTransporter) tryToConnect(nodeID string) error { } func (transporter *TCPTransporter) Publish(command, nodeID string, message moleculer.Payload) { + if command == "DISCOVER" { + if transporter.udpServer != nil { + transporter.udpServer.BroadcastDiscoveryMessage() + } + return + } + if command == "INFO" { + //how does the JS TCP transporter handle node info broadcast? + //prob done by the gossip protocol + return + } + if command == "HEARTBEAT" { + //how does the JS TCP transporter handle HEARTBEAT? + //prob done by the gossip protocol + return + } + msgType := commandToMsgType(command) if msgType == -1 { transporter.logger.Error("TCPTransporter.Publish() Invalid command: " + command + " nodeID: " + nodeID) return } - msgBts := transporter.serializer.PayloadToBytes(message) if nodeID == "" { diff --git a/transit/tcp/udp.go b/transit/tcp/udp.go index 7498ca2..5d6f37d 100644 --- a/transit/tcp/udp.go +++ b/transit/tcp/udp.go @@ -9,6 +9,7 @@ import ( "golang.org/x/net/ipv4" + "github.com/moleculer-go/moleculer" log "github.com/sirupsen/logrus" ) @@ -28,6 +29,7 @@ type UdpServer struct { discoverTimer *time.Ticker servers []*UdpServerEntry onUdpMessage OnUdpMessage + registry moleculer.Registry } type UdpServerOptions struct { @@ -43,10 +45,11 @@ type UdpServerOptions struct { NodeID string } -func NewUdpServer(opts UdpServerOptions, onUdpMessage OnUdpMessage, logger *log.Entry) *UdpServer { +func NewUdpServer(opts UdpServerOptions, registry moleculer.Registry, onUdpMessage OnUdpMessage, logger *log.Entry) *UdpServer { return &UdpServer{ opts: opts, onUdpMessage: onUdpMessage, + registry: registry, logger: logger, } } @@ -97,12 +100,12 @@ func (u *UdpServer) joinMulticastGroup(multicast string, udpConn *net.UDPConn, m for _, iface := range interfaces { if err := packetConn.JoinGroup(&iface, groupAddr); err != nil { - u.logger.Warnf("Unable to join multicast group on interface %s: %s\n", iface.Name, err) + u.logger.Tracef("Unable to join multicast group on interface %s: %s\n", iface.Name, err) } } if err := packetConn.SetMulticastTTL(multicastTTL); err != nil { - u.logger.Warnf("Unable to set multicast TTL: %s\n", err) + u.logger.Tracef("Unable to set multicast TTL: %s\n", err) return nil, err } @@ -218,7 +221,7 @@ func (u *UdpServer) getBroadcastAddresses() []string { func (u *UdpServer) firstDiscoveryMessage() { //wait for 1 second before sending the first discovery message time.Sleep(time.Second) - u.broadcastDiscoveryMessage() + u.BroadcastDiscoveryMessage() } func (u *UdpServer) handleIncomingMessagesForServer(server *UdpServerEntry) { @@ -270,7 +273,7 @@ func (u *UdpServer) startDiscovering() { u.discoverTimer = time.NewTicker(u.opts.DiscoverPeriod * time.Second) go func() { for range u.discoverTimer.C { - u.broadcastDiscoveryMessage() + u.BroadcastDiscoveryMessage() if u.opts.MaxDiscovery > 0 && u.discoveryCounter >= u.opts.MaxDiscovery { u.logger.Info("Discovery limit reached, stopping UDP discovery") u.StopDiscovering() @@ -279,8 +282,9 @@ func (u *UdpServer) startDiscovering() { }() } -func (u *UdpServer) broadcastDiscoveryMessage() { - message := fmt.Sprintf("%s|%s|%d", u.opts.Namespace, u.opts.NodeID, u.opts.Port) +func (u *UdpServer) BroadcastDiscoveryMessage() { + node := u.registry.GetLocalNode() + message := fmt.Sprintf("%s|%s|%d", u.opts.Namespace, node.GetID(), node.GetPort()) u.logger.Debug("Broadcasting discovery message:", message) u.discoveryCounter++ for _, server := range u.servers { From 37080e56cec9d31dfadac811b43063f0af9d8ece Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Thu, 11 Apr 2024 21:13:04 +1200 Subject: [PATCH 15/20] update node on gossip hello msg --- moleculer.go | 2 +- registry/registry.go | 6 +++--- transit/tcp/gossip.go | 17 ++++++++++------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/moleculer.go b/moleculer.go index 40bf076..2cb45a9 100644 --- a/moleculer.go +++ b/moleculer.go @@ -265,7 +265,7 @@ type Context interface { type ForEachNodeFunc func(node Node) bool type Registry interface { GetNodeByID(nodeID string) Node - AddOfflineNode(nodeID, address string, port int) Node + AddOfflineNode(nodeID, hostname, ipAddress string, port int) Node ForEachNode(ForEachNodeFunc) DisconnectNode(nodeID string) RemoteNodeInfoReceived(message Payload) diff --git a/registry/registry.go b/registry/registry.go index 9473c72..d77f3cb 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -116,12 +116,12 @@ func (registry *ServiceRegistry) LocalNode() moleculer.Node { } // Register a node as offline because we don't know all information about it -func (registry *ServiceRegistry) AddOfflineNode(nodeID string, address string, port int) moleculer.Node { +func (registry *ServiceRegistry) AddOfflineNode(nodeID string, hostname, ipAddress string, port int) moleculer.Node { node := CreateNode(nodeID, false, registry.logger.WithField("Node", nodeID)) node.UpdateInfo(map[string]interface{}{ - "hostname": address, + "hostname": hostname, "port": port, - "ipList": []string{address}, + "ipList": []string{ipAddress}, }) registry.nodes.Add(node) return node diff --git a/transit/tcp/gossip.go b/transit/tcp/gossip.go index 2b1de51..d79ee54 100644 --- a/transit/tcp/gossip.go +++ b/transit/tcp/gossip.go @@ -67,21 +67,24 @@ func (transporter *TCPTransporter) sendGossipToRandomEndpoint(packet moleculer.P return } node := nodes[rand.Intn(len(nodes))] - transporter.logger.Debug("Sending gossip request to " + node.GetID()) - transporter.Publish(msgTypeToCommand(PACKET_GOSSIP_REQ), node.GetID(), packet) + if !node.IsLocal() { + transporter.logger.Debug("Sending gossip request to " + node.GetID()) + transporter.Publish(msgTypeToCommand(PACKET_GOSSIP_REQ), node.GetID(), packet) + } } func (transporter *TCPTransporter) onGossipHello(fromAddrss string, msgBytes *[]byte) { - packet := transporter.serializer.BytesToPayload(msgBytes) - payload := packet.Get("payload") + payload := transporter.serializer.BytesToPayload(msgBytes) sender := payload.Get("sender").String() + port := payload.Get("port").Int() + hostname := payload.Get("host").String() - transporter.logger.Debug("Received gossip hello from " + sender) + transporter.logger.Debug("Received gossip hello from sender: ", sender, "ipAddress: ", fromAddrss, " hostname: ", hostname) node := transporter.registry.GetNodeByID(sender) if node == nil { - // Unknown node. Register as offline node - node = transporter.registry.AddOfflineNode(sender, payload.Get("host").String(), payload.Get("port").Int()) + transporter.logger.Debug("Unknown node. Register as offline node - sender: ", sender) + node = transporter.registry.AddOfflineNode(sender, hostname, fromAddrss, port) } if node.GetUdpAddress() == "" { node.UpdateInfo(map[string]interface{}{ From 82f23af8c0ecb6dfb0c684c6005a5257d0987106 Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Thu, 11 Apr 2024 21:45:50 +1200 Subject: [PATCH 16/20] basic gossip flow --- registry/node.go | 3 +-- transit/tcp/gossip.go | 17 +++++++---------- transit/tcp/tcp-reader.go | 10 +++++++--- transit/tcp/tcp-transporter.go | 10 +++++----- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/registry/node.go b/registry/node.go index d6d2ce3..9fc40e6 100644 --- a/registry/node.go +++ b/registry/node.go @@ -102,8 +102,7 @@ func (node *Node) GetPort() int { } func (node *Node) UpdateInfo(info map[string]interface{}) []map[string]interface{} { - node.logger.Debug("node.UpdateInfo() - info:") - node.logger.Debug(util.PrettyPrintMap(info)) + node.logger.Trace("node.UpdateInfo() - info:", util.PrettyPrintMap(info)) if ipListArray, ok := info["ipList"].([]interface{}); ok { node.ipList = interfaceToString(ipListArray) diff --git a/transit/tcp/gossip.go b/transit/tcp/gossip.go index d79ee54..443d2f4 100644 --- a/transit/tcp/gossip.go +++ b/transit/tcp/gossip.go @@ -41,8 +41,7 @@ func (transporter *TCPTransporter) sendGossipRequest() { }) payload := payloadPkg.Empty() - packet := payloadPkg.Empty() - packet.Add("payload", payload) + payload.Add("sender", node.GetID()) if len(onlineResponse) > 0 { payload.Add("online", onlineResponse) } @@ -51,25 +50,25 @@ func (transporter *TCPTransporter) sendGossipRequest() { } if len(onlineResponse) > 0 { - transporter.sendGossipToRandomEndpoint(packet, onlineNodes) + transporter.sendGossipToRandomEndpoint(payload, onlineNodes) } if len(offlineNodes) > 0 { ratio := float64(len(offlineNodes)) / float64(len(onlineNodes)+1) if ratio >= 1 || rand.Float64() < ratio { - transporter.sendGossipToRandomEndpoint(packet, offlineNodes) + transporter.sendGossipToRandomEndpoint(payload, offlineNodes) } } } -func (transporter *TCPTransporter) sendGossipToRandomEndpoint(packet moleculer.Payload, nodes []moleculer.Node) { +func (transporter *TCPTransporter) sendGossipToRandomEndpoint(payload moleculer.Payload, nodes []moleculer.Node) { if len(nodes) == 0 { return } node := nodes[rand.Intn(len(nodes))] if !node.IsLocal() { transporter.logger.Debug("Sending gossip request to " + node.GetID()) - transporter.Publish(msgTypeToCommand(PACKET_GOSSIP_REQ), node.GetID(), packet) + transporter.Publish(msgTypeToCommand(PACKET_GOSSIP_REQ), node.GetID(), payload) } } @@ -94,8 +93,7 @@ func (transporter *TCPTransporter) onGossipHello(fromAddrss string, msgBytes *[] } func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { - packet := transporter.serializer.BytesToPayload(msgBytes) - payload := packet.Get("payload") + payload := transporter.serializer.BytesToPayload(msgBytes) sender := payload.Get("sender").String() transporter.logger.Debug("Received gossip request from " + sender) @@ -212,8 +210,7 @@ func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { } func (transporter *TCPTransporter) onGossipResponse(msgBytes *[]byte) { - packet := transporter.serializer.BytesToPayload(msgBytes) - payload := packet.Get("payload") + payload := transporter.serializer.BytesToPayload(msgBytes) sender := payload.Get("sender").String() transporter.logger.Debug("Received gossip response from " + sender) diff --git a/transit/tcp/tcp-reader.go b/transit/tcp/tcp-reader.go index 47d5778..9508ef8 100644 --- a/transit/tcp/tcp-reader.go +++ b/transit/tcp/tcp-reader.go @@ -83,8 +83,12 @@ func (r *TcpReader) Listen() (int, error) { func (r *TcpReader) handleConnection(conn net.Conn) { address := conn.RemoteAddr().String() + host, _, err := net.SplitHostPort(address) + if err != nil { + r.logger.Error("Failed to split host and port - address:", address) + } + r.logger.Debugf("New TCP client connected from '%s'\n", address) - var err error for err == nil { msgType, msgBytes, e := r.readMessage(conn) err = e @@ -92,8 +96,8 @@ func (r *TcpReader) handleConnection(conn net.Conn) { r.logger.Errorf("Error reading message from '%s': %s", address, err) break } - r.logger.Debug("handleConnection() message read from socket - msgType: ", msgType, "message:", string(msgBytes)) - r.onMessage(address, msgType, &msgBytes) + r.logger.Trace("handleConnection() message read from socket - msgType: ", msgType, "message:", string(msgBytes)) + r.onMessage(host, msgType, &msgBytes) } r.closeSocket(conn) } diff --git a/transit/tcp/tcp-transporter.go b/transit/tcp/tcp-transporter.go index d86f45c..c335ef6 100644 --- a/transit/tcp/tcp-transporter.go +++ b/transit/tcp/tcp-transporter.go @@ -255,14 +255,14 @@ func (transporter *TCPTransporter) onUdpMessage(nodeID, address string, port int transporter.logger.Debug("UDP discovery received from " + address + " nodeId: " + nodeID + " port: " + string(port)) node := transporter.registry.GetNodeByID(nodeID) if node == nil { - // Unknown node. Register as offline node - node = transporter.registry.AddOfflineNode(nodeID, address, port) + transporter.logger.Debug("Unknown node. Register as offline node") + node = transporter.registry.AddOfflineNode(nodeID, address, address, port) } else if !node.IsAvailable() { ipList := addIpToList(node.GetIpList(), address) node.UpdateInfo(map[string]interface{}{ - "hostname": address, - "port": port, - "ipList": ipList, + // "hostname": address, + "port": port, + "ipList": ipList, }) } node.UpdateInfo(map[string]interface{}{ From 8afcaf860f0c7cbd2ed83fffc0dc526bee89057f Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Thu, 11 Apr 2024 23:06:55 +1200 Subject: [PATCH 17/20] basic discovery flow and protocol --- registry/node.go | 1 + transit/pubsub/pubsub.go | 28 ++++++++++------------ transit/tcp/gossip.go | 43 ++++++++++++++++++++++++---------- transit/tcp/tcp-transporter.go | 10 ++++---- 4 files changed, 49 insertions(+), 33 deletions(-) diff --git a/registry/node.go b/registry/node.go index 9fc40e6..bf191da 100644 --- a/registry/node.go +++ b/registry/node.go @@ -304,4 +304,5 @@ func (node *Node) Available() { func (node *Node) IncreaseSequence() { node.sequence++ + node.logger.Debug("node.IncreaseSequence() nodeID:", node.id, "sequence:", node.sequence) } diff --git a/transit/pubsub/pubsub.go b/transit/pubsub/pubsub.go index 38ac261..33b3966 100644 --- a/transit/pubsub/pubsub.go +++ b/transit/pubsub/pubsub.go @@ -48,23 +48,21 @@ const DATATYPE_JSON = 2 const DATATYPE_BUFFER = 3 func (pubsub *PubSub) onServiceAdded(values ...interface{}) { - if pubsub.isConnected && pubsub.brokerStarted { - localNodeID := pubsub.broker.LocalNode().GetID() - - // Checking that was added local service - isLocalServiceAdded := false - for _, value := range values { - if value.(map[string]string)["nodeID"] == localNodeID { - isLocalServiceAdded = true - break - } - } - - if isLocalServiceAdded { - pubsub.broker.LocalNode().IncreaseSequence() - pubsub.broadcastNodeInfo("") + localNodeID := pubsub.broker.LocalNode().GetID() + // Checking that was added local service + isLocalServiceAdded := false + for _, value := range values { + if value.(map[string]string)["nodeID"] == localNodeID { + isLocalServiceAdded = true + break } } + if isLocalServiceAdded { + pubsub.broker.LocalNode().IncreaseSequence() + } + if isLocalServiceAdded && pubsub.isConnected && pubsub.brokerStarted { + pubsub.broadcastNodeInfo("") + } } func (pubsub *PubSub) onBrokerStarted(values ...interface{}) { diff --git a/transit/tcp/gossip.go b/transit/tcp/gossip.go index 443d2f4..9364966 100644 --- a/transit/tcp/gossip.go +++ b/transit/tcp/gossip.go @@ -6,6 +6,7 @@ import ( "github.com/moleculer-go/moleculer" payloadPkg "github.com/moleculer-go/moleculer/payload" + "github.com/moleculer-go/moleculer/util" ) func (transporter *TCPTransporter) startGossipTimer() { @@ -19,7 +20,7 @@ func (transporter *TCPTransporter) startGossipTimer() { func (transporter *TCPTransporter) sendGossipRequest() { - transporter.logger.Debug("Sending gossip request") + transporter.logger.Trace("Sending gossip request") node := transporter.registry.GetLocalNode() node.UpdateMetrics() @@ -67,7 +68,7 @@ func (transporter *TCPTransporter) sendGossipToRandomEndpoint(payload moleculer. } node := nodes[rand.Intn(len(nodes))] if !node.IsLocal() { - transporter.logger.Debug("Sending gossip request to " + node.GetID()) + transporter.logger.Trace("Sending gossip request to "+node.GetID(), "payload:", payload) transporter.Publish(msgTypeToCommand(PACKET_GOSSIP_REQ), node.GetID(), payload) } } @@ -121,19 +122,29 @@ func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { if onlineMap.Exists() { online = onlineMap.Get(node.GetID()) if online.Exists() { - transporter.logger.Debug("received seq, cpuSeq, cpu for " + node.GetID()) + transporter.logger.Debug("received for: "+node.GetID(), " seq: ", seq, " cpuSeq: ", cpuSeq, " cpu: ", cpu) seq = online.Get("seq").Int64() cpuSeq = online.Get("cpuSeq").Int64() cpu = online.Get("cpu").Int64() } } + if node.IsLocal() { + node.UpdateMetrics() + info := node.ExportAsMap() + seq = node.GetSequence() + cpu = node.GetCpu() + cpuSeq = node.GetCpuSequence() + onlineResponse[node.GetID()] = []interface{}{info, node.GetCpuSequence(), node.GetCpu()} + // transporter.logger.Debug("Node is local - send back the node info and cpu, cpuSed to "+node.GetID(), " seq: ", seq, " cpuSeq: ", cpuSeq, " cpu: ", cpu, " info: ", util.PrettyPrintMap(info)) + } + if seq != 0 && seq < node.GetSequence() { transporter.logger.Debug("We have newer info or requester doesn't know it") if node.IsAvailable() { info := node.ExportAsMap() onlineResponse[node.GetID()] = []interface{}{info, node.GetCpuSequence(), node.GetCpu()} - transporter.logger.Debug("Node is available - send back the node info and cpu, cpuSed to " + node.GetID()) + transporter.logger.Debug("Node is available - send back the node info and cpu, cpuSed to "+node.GetID(), " seq: ", seq, " cpuSeq: ", cpuSeq, " cpu: ", cpu, " info: ", util.PrettyPrintMap(info)) } else { offlineResponse[node.GetID()] = node.GetSequence() transporter.logger.Debug("Node is offline - send back the seq to " + node.GetID()) @@ -193,6 +204,7 @@ func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { } } else { // We know it as offline. We do nothing, because we'll request it and we'll receive its INFO. + return true } } @@ -201,10 +213,15 @@ func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { if len(onlineResponse) > 0 || len(offlineResponse) > 0 { sender := payload.Get("sender").String() - transporter.Publish(msgTypeToCommand(PACKET_GOSSIP_RES), sender, payloadPkg.Empty().Add("online", onlineResponse).Add("offline", offlineResponse)) - transporter.logger.Debug("Gossip response sent to " + sender) + responsePayload := payloadPkg. + Empty(). + Add("online", onlineResponse). + Add("offline", offlineResponse). + Add("sender", transporter.registry.GetLocalNode().GetID()) + transporter.Publish(msgTypeToCommand(PACKET_GOSSIP_RES), sender, responsePayload) + transporter.logger.Trace("Gossip response sent to " + sender) } else { - transporter.logger.Debug("No response sent to " + payload.Get("sender").String()) + transporter.logger.Trace("No response sent to " + sender) } } @@ -234,10 +251,10 @@ func (transporter *TCPTransporter) onGossipResponse(msgBytes *[]byte) { row := online.Get(nodeID).Array() info, cpu, cpuSeq := parseGossipResponse(row) - if info != nil && (node != nil || node.GetSequence() < info["seq"].(int64)) { + if info != nil && (node != nil && node.GetSequence() < info.Get("seq").Int64()) { transporter.logger.Debug("If we don't know it, or know, but has smaller seq, update 'info'") - info["sender"] = sender - transporter.registry.RemoteNodeInfoReceived(payloadPkg.New(info)) + info = info.Add("sender", sender) + transporter.registry.RemoteNodeInfoReceived(info) } if node != nil && cpuSeq > node.GetCpuSequence() { transporter.logger.Debug("If we know it and has smaller cpuSeq, update 'cpu'") @@ -283,18 +300,18 @@ func (transporter *TCPTransporter) onGossipResponse(msgBytes *[]byte) { } } -func parseGossipResponse(row []moleculer.Payload) (info map[string]interface{}, cpu int64, cpuSeq int64) { +func parseGossipResponse(row []moleculer.Payload) (info moleculer.Payload, cpu int64, cpuSeq int64) { cpuSeq = -1 cpu = -1 if len(row) == 1 { - info = row[0].RawMap() + info = row[0] } if len(row) == 2 { cpuSeq = row[0].Int64() cpu = row[1].Int64() } if len(row) == 3 { - info = row[0].RawMap() + info = row[0] cpuSeq = row[1].Int64() cpu = row[2].Int64() } diff --git a/transit/tcp/tcp-transporter.go b/transit/tcp/tcp-transporter.go index c335ef6..493429a 100644 --- a/transit/tcp/tcp-transporter.go +++ b/transit/tcp/tcp-transporter.go @@ -180,13 +180,13 @@ func (transporter *TCPTransporter) incomingMessage(msgType int, msgBytes *[]byte return } message := transporter.serializer.BytesToPayload(msgBytes) - if transporter.validateMsg(message) { - if handlers, ok := transporter.handlers[command]; ok { - for _, handler := range handlers { - handler(message) - } + // if transporter.validateMsg(message) { + if handlers, ok := transporter.handlers[command]; ok { + for _, handler := range handlers { + handler(message) } } + // } } func (transporter *TCPTransporter) startTcpServer() { From 3cecc1e851290dd503b5d96a6229b2f13a612b9c Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Fri, 12 Apr 2024 20:05:39 +1200 Subject: [PATCH 18/20] manage disconnection --- moleculer.go | 2 +- registry/node.go | 8 +++---- registry/nodeCatalog.go | 22 ++++++++++++++++++++ registry/registry.go | 4 ++++ transit/tcp/gossip.go | 8 +++---- transit/tcp/tcp-reader.go | 38 +++++++++++++++++++++------------- transit/tcp/tcp-transporter.go | 10 ++++++++- 7 files changed, 68 insertions(+), 24 deletions(-) diff --git a/moleculer.go b/moleculer.go index 2cb45a9..26c156d 100644 --- a/moleculer.go +++ b/moleculer.go @@ -240,7 +240,6 @@ type Node interface { GetCpuSequence() int64 GetCpu() int64 IsLocal() bool - Disconnected(isUnexpected bool) UpdateMetrics() GetHostname() string } @@ -270,6 +269,7 @@ type Registry interface { DisconnectNode(nodeID string) RemoteNodeInfoReceived(message Payload) GetLocalNode() Node + GetNodeByHost(host string) Node } type BrokerContext interface { diff --git a/registry/node.go b/registry/node.go index bf191da..1f8df2e 100644 --- a/registry/node.go +++ b/registry/node.go @@ -240,10 +240,6 @@ func interfaceToString(list []interface{}) []string { return result } -func (node *Node) Disconnected(isUnexpected bool) { - -} - // ExportAsMap export the node info as a map // this map is used to publish the node info to other nodes. func (node *Node) ExportAsMap() map[string]interface{} { @@ -294,6 +290,10 @@ func (node *Node) IsAvailable() bool { // Unavailable mark the node as unavailable func (node *Node) Unavailable() { + if node.isAvailable { + node.offlineSince = time.Now().Unix() + node.sequence++ + } node.isAvailable = false } diff --git a/registry/nodeCatalog.go b/registry/nodeCatalog.go index 7a9bb2b..5a899ca 100644 --- a/registry/nodeCatalog.go +++ b/registry/nodeCatalog.go @@ -19,6 +19,28 @@ func CreateNodesCatalog(logger *log.Entry) *NodeCatalog { return &NodeCatalog{sync.Map{}, logger} } +func contains(str string, list []string) bool { + for _, v := range list { + if v == str { + return true + } + } + return false +} + +func (catalog *NodeCatalog) GetNodeByHost(host string) moleculer.Node { + var result moleculer.Node + catalog.nodes.Range(func(key, value interface{}) bool { + node := value.(moleculer.Node) + if contains(host, node.GetIpList()) || node.GetHostname() == host { + result = node + return false + } + return true + }) + return result +} + // HeartBeat delegate the heart beat to the node in question payload.sender func (catalog *NodeCatalog) HeartBeat(heartbeat map[string]interface{}) bool { sender := heartbeat["sender"].(string) diff --git a/registry/registry.go b/registry/registry.go index d77f3cb..7147f9c 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -593,6 +593,10 @@ func (registry *ServiceRegistry) nextAction(actionName string, strategy strategy return registry.actions.Next(actionName, strategy) } +func (registry *ServiceRegistry) GetNodeByHost(ip string) moleculer.Node { + return registry.nodes.GetNodeByHost(ip) +} + func (registry *ServiceRegistry) KnownEventListeners(addNode bool) []string { events := registry.events.list() result := make([]string, len(events)) diff --git a/transit/tcp/gossip.go b/transit/tcp/gossip.go index 9364966..ef4c725 100644 --- a/transit/tcp/gossip.go +++ b/transit/tcp/gossip.go @@ -97,7 +97,7 @@ func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { payload := transporter.serializer.BytesToPayload(msgBytes) sender := payload.Get("sender").String() - transporter.logger.Debug("Received gossip request from " + sender) + transporter.logger.Trace("Received gossip request from " + sender) onlineResponse := map[string]interface{}{} offlineResponse := map[string]interface{}{} @@ -122,7 +122,7 @@ func (transporter *TCPTransporter) onGossipRequest(msgBytes *[]byte) { if onlineMap.Exists() { online = onlineMap.Get(node.GetID()) if online.Exists() { - transporter.logger.Debug("received for: "+node.GetID(), " seq: ", seq, " cpuSeq: ", cpuSeq, " cpu: ", cpu) + transporter.logger.Trace("received for: "+node.GetID(), " seq: ", seq, " cpuSeq: ", cpuSeq, " cpu: ", cpu) seq = online.Get("seq").Int64() cpuSeq = online.Get("cpuSeq").Int64() cpu = online.Get("cpu").Int64() @@ -230,13 +230,13 @@ func (transporter *TCPTransporter) onGossipResponse(msgBytes *[]byte) { payload := transporter.serializer.BytesToPayload(msgBytes) sender := payload.Get("sender").String() - transporter.logger.Debug("Received gossip response from " + sender) + transporter.logger.Trace("Received gossip response from " + sender) online := payload.Get("online") offline := payload.Get("offline") if online.Exists() { - transporter.logger.Debug("Received online info from nodeID: " + sender) + transporter.logger.Trace("Received online info from nodeID: " + sender) online.ForEach(func(key interface{}, value moleculer.Payload) bool { nodeID, ok := key.(string) if !ok { diff --git a/transit/tcp/tcp-reader.go b/transit/tcp/tcp-reader.go index 9508ef8..eaff24c 100644 --- a/transit/tcp/tcp-reader.go +++ b/transit/tcp/tcp-reader.go @@ -20,22 +20,24 @@ const ( type OnMessageFunc func(fromAddrss string, msgType int, msgBytes *[]byte) type TcpReader struct { - port int - listener net.Listener - sockets map[net.Conn]bool - logger *log.Entry - lock sync.Mutex - state State - maxPacketSize int - onMessage OnMessageFunc + port int + listener net.Listener + sockets map[net.Conn]bool + logger *log.Entry + lock sync.Mutex + state State + maxPacketSize int + onMessage OnMessageFunc + disconnectNodeByHost func(host string) } -func NewTcpReader(port int, onMessage OnMessageFunc, logger *log.Entry) *TcpReader { +func NewTcpReader(port int, onMessage OnMessageFunc, disconnectNodeByHost func(host string), logger *log.Entry) *TcpReader { return &TcpReader{ port: port, sockets: make(map[net.Conn]bool), logger: logger, onMessage: onMessage, + disconnectNodeByHost : disconnectNodeByHost } } @@ -93,7 +95,13 @@ func (r *TcpReader) handleConnection(conn net.Conn) { msgType, msgBytes, e := r.readMessage(conn) err = e if err != nil { - r.logger.Errorf("Error reading message from '%s': %s", address, err) + + if err.Error() == "EOF" { + r.logger.Debugf("EOF received from '%s' ", address) + r.disconnectNodeByHost(host) + } else { + r.logger.Errorf("Error reading message from '%s': %s", address, err) + } break } r.logger.Trace("handleConnection() message read from socket - msgType: ", msgType, "message:", string(msgBytes)) @@ -106,7 +114,7 @@ func (r *TcpReader) readMessage(conn net.Conn) (msgType int, msg []byte, err err var buf []byte for { // Read data from the connection - chunk := make([]byte, 256) + chunk := make([]byte, 1024) n, err := conn.Read(chunk) if err != nil { return 0, nil, err @@ -126,14 +134,16 @@ func (r *TcpReader) readMessage(conn net.Conn) (msgType int, msg []byte, err err if r.maxPacketSize > 0 && len(buf) > r.maxPacketSize { return 0, nil, fmt.Errorf("incoming packet is larger than the 'maxPacketSize' limit (%d > %d)", len(buf), r.maxPacketSize) } + + length := int(binary.BigEndian.Uint32(buf[1:])) + // Check the CRC crc := buf[1] ^ buf[2] ^ buf[3] ^ buf[4] ^ buf[5] if crc != buf[0] { - return 0, nil, fmt.Errorf("invalid packet CRC %d", crc) + r.logger.Errorf("invalid packet CRC: %d buf[0]: %d buf: %s", crc, buf[0], string(buf)) + return 0, nil, fmt.Errorf("invalid packet CRC: %d buf[0]: %d ", crc, buf[0]) } - length := int(binary.BigEndian.Uint32(buf[1:])) - // If the buffer contains a complete message, return it if len(buf) >= length { msg = buf[6:length] diff --git a/transit/tcp/tcp-transporter.go b/transit/tcp/tcp-transporter.go index 493429a..3778383 100644 --- a/transit/tcp/tcp-transporter.go +++ b/transit/tcp/tcp-transporter.go @@ -179,6 +179,7 @@ func (transporter *TCPTransporter) incomingMessage(msgType int, msgBytes *[]byte transporter.logger.Error("Unknown command received - msgType: " + string(msgType)) return } + transporter.logger.Debug("Incoming message - command: " + command) message := transporter.serializer.BytesToPayload(msgBytes) // if transporter.validateMsg(message) { if handlers, ok := transporter.handlers[command]; ok { @@ -189,8 +190,15 @@ func (transporter *TCPTransporter) incomingMessage(msgType int, msgBytes *[]byte // } } +func (transporter *TCPTransporter) disconnectNodeByHost(host string) { + node := transporter.registry.GetNodeByHost(host) + if node != nil { + transporter.registry.DisconnectNode(node.GetID()) + } +} + func (transporter *TCPTransporter) startTcpServer() { - transporter.tcpReader = NewTcpReader(transporter.options.Port, transporter.onTcpMessage, transporter.logger.WithFields(log.Fields{ + transporter.tcpReader = NewTcpReader(transporter.options.Port, transporter.onTcpMessage, transporter.disconnectNodeByHost, transporter.logger.WithFields(log.Fields{ "TCPTransporter": "TCPReader", })) transporter.tcpWriter = NewTcpWriter(transporter.options.MaxConnections, transporter.logger.WithFields(log.Fields{ From 82fc3810c70f1eec7211c8d6136023d0eef73ef0 Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Fri, 12 Apr 2024 20:55:16 +1200 Subject: [PATCH 19/20] handle disconection --- moleculer.go | 2 +- registry/nodeCatalog.go | 18 ++++++++++++++++-- registry/registry.go | 4 ++-- transit/tcp/gossip.go | 22 +++++++++++++++++++--- transit/tcp/tcp-reader.go | 32 ++++++++++++++++---------------- transit/tcp/tcp-transporter.go | 13 ++++++------- 6 files changed, 60 insertions(+), 31 deletions(-) diff --git a/moleculer.go b/moleculer.go index 26c156d..f1ce25a 100644 --- a/moleculer.go +++ b/moleculer.go @@ -269,7 +269,7 @@ type Registry interface { DisconnectNode(nodeID string) RemoteNodeInfoReceived(message Payload) GetLocalNode() Node - GetNodeByHost(host string) Node + GetNodeByAddress(host string) Node } type BrokerContext interface { diff --git a/registry/nodeCatalog.go b/registry/nodeCatalog.go index 5a899ca..0d7c5af 100644 --- a/registry/nodeCatalog.go +++ b/registry/nodeCatalog.go @@ -1,6 +1,8 @@ package registry import ( + "net" + "strconv" "sync" "time" @@ -28,11 +30,23 @@ func contains(str string, list []string) bool { return false } -func (catalog *NodeCatalog) GetNodeByHost(host string) moleculer.Node { +func (catalog *NodeCatalog) GetNodeByAddress(address string) moleculer.Node { + + host, portString, err := net.SplitHostPort(address) + if err != nil { + catalog.logger.Error("GetNodeByAddress() Error parsing address: ", address) + return nil + } + port, err := strconv.Atoi(portString) + if err != nil { + catalog.logger.Error("GetNodeByAddress() Error parsing port: ", portString) + return nil + } + var result moleculer.Node catalog.nodes.Range(func(key, value interface{}) bool { node := value.(moleculer.Node) - if contains(host, node.GetIpList()) || node.GetHostname() == host { + if (contains(host, node.GetIpList()) || node.GetHostname() == host) && node.GetPort() == port { result = node return false } diff --git a/registry/registry.go b/registry/registry.go index 7147f9c..2d27318 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -593,8 +593,8 @@ func (registry *ServiceRegistry) nextAction(actionName string, strategy strategy return registry.actions.Next(actionName, strategy) } -func (registry *ServiceRegistry) GetNodeByHost(ip string) moleculer.Node { - return registry.nodes.GetNodeByHost(ip) +func (registry *ServiceRegistry) GetNodeByAddress(address string) moleculer.Node { + return registry.nodes.GetNodeByAddress(address) } func (registry *ServiceRegistry) KnownEventListeners(addNode bool) []string { diff --git a/transit/tcp/gossip.go b/transit/tcp/gossip.go index ef4c725..1b17773 100644 --- a/transit/tcp/gossip.go +++ b/transit/tcp/gossip.go @@ -13,12 +13,12 @@ func (transporter *TCPTransporter) startGossipTimer() { transporter.gossipTimer = time.NewTicker(time.Second * time.Duration(transporter.options.GossipPeriod)) go func() { for range transporter.gossipTimer.C { - transporter.sendGossipRequest() + transporter.sendGossipRequest(false) } }() } -func (transporter *TCPTransporter) sendGossipRequest() { +func (transporter *TCPTransporter) sendGossipRequest(broadcast bool) { transporter.logger.Trace("Sending gossip request") @@ -51,7 +51,11 @@ func (transporter *TCPTransporter) sendGossipRequest() { } if len(onlineResponse) > 0 { - transporter.sendGossipToRandomEndpoint(payload, onlineNodes) + if broadcast { + transporter.broadcastGossipToNodes(payload, onlineNodes) + } else { + transporter.sendGossipToRandomEndpoint(payload, onlineNodes) + } } if len(offlineNodes) > 0 { @@ -62,6 +66,18 @@ func (transporter *TCPTransporter) sendGossipRequest() { } } +func (transporter *TCPTransporter) broadcastGossipToNodes(payload moleculer.Payload, nodes []moleculer.Node) { + if len(nodes) == 0 { + return + } + for _, node := range nodes { + if !node.IsLocal() { + transporter.logger.Trace("Sending gossip request to "+node.GetID(), "payload:", payload) + transporter.Publish(msgTypeToCommand(PACKET_GOSSIP_REQ), node.GetID(), payload) + } + } +} + func (transporter *TCPTransporter) sendGossipToRandomEndpoint(payload moleculer.Payload, nodes []moleculer.Node) { if len(nodes) == 0 { return diff --git a/transit/tcp/tcp-reader.go b/transit/tcp/tcp-reader.go index eaff24c..798912e 100644 --- a/transit/tcp/tcp-reader.go +++ b/transit/tcp/tcp-reader.go @@ -20,24 +20,24 @@ const ( type OnMessageFunc func(fromAddrss string, msgType int, msgBytes *[]byte) type TcpReader struct { - port int - listener net.Listener - sockets map[net.Conn]bool - logger *log.Entry - lock sync.Mutex - state State - maxPacketSize int - onMessage OnMessageFunc - disconnectNodeByHost func(host string) + port int + listener net.Listener + sockets map[net.Conn]bool + logger *log.Entry + lock sync.Mutex + state State + maxPacketSize int + onMessage OnMessageFunc + disconnectNodeByAddress func(address string) } -func NewTcpReader(port int, onMessage OnMessageFunc, disconnectNodeByHost func(host string), logger *log.Entry) *TcpReader { +func NewTcpReader(port int, onMessage OnMessageFunc, disconnectNodeByAddress func(address string), logger *log.Entry) *TcpReader { return &TcpReader{ - port: port, - sockets: make(map[net.Conn]bool), - logger: logger, - onMessage: onMessage, - disconnectNodeByHost : disconnectNodeByHost + port: port, + sockets: make(map[net.Conn]bool), + logger: logger, + onMessage: onMessage, + disconnectNodeByAddress: disconnectNodeByAddress, } } @@ -98,7 +98,7 @@ func (r *TcpReader) handleConnection(conn net.Conn) { if err.Error() == "EOF" { r.logger.Debugf("EOF received from '%s' ", address) - r.disconnectNodeByHost(host) + r.disconnectNodeByAddress(address) } else { r.logger.Errorf("Error reading message from '%s': %s", address, err) } diff --git a/transit/tcp/tcp-transporter.go b/transit/tcp/tcp-transporter.go index 3778383..47c8aa0 100644 --- a/transit/tcp/tcp-transporter.go +++ b/transit/tcp/tcp-transporter.go @@ -190,15 +190,15 @@ func (transporter *TCPTransporter) incomingMessage(msgType int, msgBytes *[]byte // } } -func (transporter *TCPTransporter) disconnectNodeByHost(host string) { - node := transporter.registry.GetNodeByHost(host) - if node != nil { +func (transporter *TCPTransporter) disconnectNodeByAddress(address string) { + node := transporter.registry.GetNodeByAddress(address) + if node != nil && !node.IsLocal() { transporter.registry.DisconnectNode(node.GetID()) } } func (transporter *TCPTransporter) startTcpServer() { - transporter.tcpReader = NewTcpReader(transporter.options.Port, transporter.onTcpMessage, transporter.disconnectNodeByHost, transporter.logger.WithFields(log.Fields{ + transporter.tcpReader = NewTcpReader(transporter.options.Port, transporter.onTcpMessage, transporter.disconnectNodeByAddress, transporter.logger.WithFields(log.Fields{ "TCPTransporter": "TCPReader", })) transporter.tcpWriter = NewTcpWriter(transporter.options.MaxConnections, transporter.logger.WithFields(log.Fields{ @@ -345,13 +345,12 @@ func (transporter *TCPTransporter) Publish(command, nodeID string, message molec return } if command == "INFO" { - //how does the JS TCP transporter handle node info broadcast? - //prob done by the gossip protocol + transporter.sendGossipRequest(true) return } if command == "HEARTBEAT" { //how does the JS TCP transporter handle HEARTBEAT? - //prob done by the gossip protocol + //prob done by the gossip protocol - already has a timer return } From f00ed362f8b2913b62352498cbec2ae3baed463c Mon Sep 17 00:00:00 2001 From: Rafael Almeida Date: Fri, 12 Apr 2024 21:32:11 +1200 Subject: [PATCH 20/20] fix merge config --- broker/broker.go | 69 +++++++++++++++++++++++++++------- moleculer.go | 4 +- transit/tcp/tcp-transporter.go | 1 + 3 files changed, 58 insertions(+), 16 deletions(-) diff --git a/broker/broker.go b/broker/broker.go index bf1de2a..77e7487 100644 --- a/broker/broker.go +++ b/broker/broker.go @@ -55,33 +55,71 @@ func mergeConfigs(baseConfig moleculer.Config, userConfig []*moleculer.Config) m if config.StrategyFactory != nil { baseConfig.StrategyFactory = config.StrategyFactory } - if config.DisableInternalMiddlewares { - baseConfig.DisableInternalMiddlewares = config.DisableInternalMiddlewares + if config.UpdateNodeMetricsFrequency != 0 { + baseConfig.UpdateNodeMetricsFrequency = config.UpdateNodeMetricsFrequency } - if config.DisableInternalServices { - baseConfig.DisableInternalServices = config.DisableInternalServices + if config.HeartbeatFrequency != 0 { + baseConfig.HeartbeatFrequency = config.HeartbeatFrequency + } + if config.HeartbeatTimeout != 0 { + baseConfig.HeartbeatTimeout = config.HeartbeatTimeout + } + if config.OfflineCheckFrequency != 0 { + baseConfig.OfflineCheckFrequency = config.OfflineCheckFrequency + } + if config.OfflineTimeout != 0 { + baseConfig.OfflineTimeout = config.OfflineTimeout + } + if config.NeighboursCheckTimeout != 0 { + baseConfig.NeighboursCheckTimeout = config.NeighboursCheckTimeout + } + if config.WaitForDependenciesTimeout != 0 { + baseConfig.WaitForDependenciesTimeout = config.WaitForDependenciesTimeout + } + if config.Middlewares != nil { + baseConfig.Middlewares = config.Middlewares + } + if config.Namespace != "" { + baseConfig.Namespace = config.Namespace + } + if config.RequestTimeout != 0 { + baseConfig.RequestTimeout = config.RequestTimeout + } + if config.MCallTimeout != 0 { + baseConfig.MCallTimeout = config.MCallTimeout + } + if config.RetryPolicy != nil { + baseConfig.RetryPolicy = config.RetryPolicy + } + if config.MaxCallLevel != 0 { + baseConfig.MaxCallLevel = config.MaxCallLevel } if config.Metrics { baseConfig.Metrics = config.Metrics } - if config.MetricsRate > 0 { baseConfig.MetricsRate = config.MetricsRate } - + if config.DisableInternalServices { + baseConfig.DisableInternalServices = config.DisableInternalServices + } + if config.DisableInternalMiddlewares { + baseConfig.DisableInternalMiddlewares = config.DisableInternalMiddlewares + } if config.DontWaitForNeighbours { baseConfig.DontWaitForNeighbours = config.DontWaitForNeighbours } - - if config.Middlewares != nil { - baseConfig.Middlewares = config.Middlewares + if config.WaitForNeighboursInterval != 0 { + baseConfig.WaitForNeighboursInterval = config.WaitForNeighboursInterval } - if config.RequestTimeout != 0 { - baseConfig.RequestTimeout = config.RequestTimeout + if config.Created != nil { + baseConfig.Created = config.Created } - - if config.Namespace != "" { - baseConfig.Namespace = config.Namespace + if config.Started != nil { + baseConfig.Started = config.Started + } + if config.Stopped != nil { + baseConfig.Stopped = config.Stopped } } } @@ -330,6 +368,9 @@ func (broker *ServiceBroker) waitForService(service string) error { break } if time.Since(start) > broker.config.WaitForDependenciesTimeout { + broker.logger.Debug("Time:", time.Since(start)) + broker.logger.Debug("WaitForDependenciesTimeout:", broker.config.WaitForDependenciesTimeout) + err := errors.New("waitForService() - Timeout ! service: " + service) broker.logger.Error(err) return err diff --git a/moleculer.go b/moleculer.go index f1ce25a..85c99e2 100644 --- a/moleculer.go +++ b/moleculer.go @@ -134,7 +134,7 @@ type Config struct { Namespace string RequestTimeout time.Duration MCallTimeout time.Duration - RetryPolicy RetryPolicy + RetryPolicy *RetryPolicy MaxCallLevel int Metrics bool MetricsRate float32 @@ -170,7 +170,7 @@ var DefaultConfig = Config{ Started: func() {}, Stopped: func() {}, MaxCallLevel: 100, - RetryPolicy: RetryPolicy{ + RetryPolicy: &RetryPolicy{ Enabled: false, }, RequestTimeout: 3 * time.Second, diff --git a/transit/tcp/tcp-transporter.go b/transit/tcp/tcp-transporter.go index 47c8aa0..bb771a8 100644 --- a/transit/tcp/tcp-transporter.go +++ b/transit/tcp/tcp-transporter.go @@ -338,6 +338,7 @@ func (transporter *TCPTransporter) tryToConnect(nodeID string) error { } func (transporter *TCPTransporter) Publish(command, nodeID string, message moleculer.Payload) { + transporter.logger.Debug("TCPTransporter.Publish() command: " + command + " to nodeID: " + nodeID) if command == "DISCOVER" { if transporter.udpServer != nil { transporter.udpServer.BroadcastDiscoveryMessage()