Mysync doesn't start when incorrect zk DNS is specified

[noname0443]

If we specify an incorrect zk DNS in the Mysync configuration or delete a zk host along with its DNS record, Mysync will not start. For an example, I delete zoo3 from docker-compose.yaml and get an error that server zoo3 is misbehaving. This happens because we immediately throw an error from the function if we cannot get the host's IP using DNS in the function.

	for _, server := range srvs {
		ips, err := addrsByHostname(server)
		if err != nil {
			return nil, err
		}
		for _, ip := range ips {
			addrToHostname[ip] = server
		}
	}

Is it a bug or a feature? :)
I believe this is not a common occurrence, and in most cases, it can help prevent users from specifying invalid DNS records. However, in the rare cases when a user deletes a host along with its DNS and restarts Mysync, they will encounter this error. I applied Mysync to my project using Kubernetes+Helm and encountered this issue when scaling down the Zookeeper cluster.

Configurations

version: '2.3'

services:
  zoo1:
    build:
      context: ./zookeeper
      args:
        - VERSION=$VERSION
    hostname: zoo1
    ports:
      - 2181
      - 2281
      - 2888
      - 3888
    privileged: true
    environment:
      ZK_SERVERS: server.1=zoo1:2888:3888, server.2=zoo2:2888:3888, server.3=zoo3:2888:3888
      ZK_MYID: 1
    healthcheck:
      test: "nc -z localhost 2181"
      start_period: 30s
      timeout: 5s
      interval: 10s
      retries: 3
    networks:
      mysync_net:
        ipv4_address: 192.168.233.10

  zoo2:
    build:
      context: ./zookeeper
      args:
        - VERSION=$VERSION
    hostname: zoo2
    ports:
      - 2181
      - 2281
      - 2888
      - 3888
    privileged: true
    environment:
      ZK_SERVERS: server.1=zoo1:2888:3888, server.2=zoo2:2888:3888, server.3=zoo3:2888:3888
      ZK_MYID: 2
    networks:
      mysync_net:
        ipv4_address: 192.168.233.11

  #zoo3:
  #  build:
  #    context: ./zookeeper
  #    args:
  #      - VERSION=$VERSION
  #  hostname: zoo3
  #  ports:
  #    - 2181
  #    - 2281
  #    - 2888
  #    - 3888
  #  privileged: true
  #  environment:
  #    ZK_SERVERS: server.1=zoo1:2888:3888, server.2=zoo2:2888:3888, server.3=zoo3:2888:3888
  #    ZK_MYID: 3
  #  networks:
  #    mysync_net:
  #      ipv4_address: 192.168.233.12

  mysql1:
    build:
      context: ./mysql
      args:
        - VERSION=$VERSION
    hostname: mysql1
    ports:
        - 3306
    privileged: true
    environment:
      MYSQL_SERVER_ID: 1
      MYSQL_ADMIN_USER: admin
      MYSQL_ADMIN_PASSWORD: admin_pwd
      MYSQL_MASTER: ''
      MYSYNC_FAILOVER:
      MYSYNC_FAILOVER_DELAY:
      MYSYNC_FAILOVER_COOLDOWN:
      MYSYNC_INACTIVATION_DELAY:
      ZK_SERVERS: zoo1:2281, zoo2:2281, zoo3:2281
      MYSQL_PORT: 3306
      MYSYNC_DISABLE_REPLICATION_ON_MAINT: "true"
      MYSYNC_SEMISYNC:
      MYSYNC_ASYNC:
      ASYNC_ALLOWED_LAG:
      MYSYNC_CRITICAL_DISK_USAGE:
      MYSYNC_KEEP_SUPER_WRITABLE_ON_CRITICAL_DISK_USAGE:
      MYSYNC_WAIT_FOR_SLAVE_COUNT:
      MYSYNC_STREAM_FROM_REASONABLE_LAG:
      MYSYNC_RESETUP_CRASHED_HOSTS:
      MYSYNC_REPLICATION_REPAIR_AGGRESSIVE_MODE:
      MYSYNC_SET_RO_TIMEOUT:
      MYSYNC_REPLICATION_LAG_QUERY:
      REPL_MON:
    healthcheck:
      test: "mysql --user=admin --password=admin_pwd -e 'SELECT 1'"
      start_period: 30s
      timeout: 5s
      interval: 10s
      retries: 3
    depends_on:
      zoo1:
        condition: service_healthy
    networks:
      mysync_net:
        ipv4_address: 192.168.233.13

  mysql2:
    build:
      context: ./mysql
      args:
        - VERSION=$VERSION
    hostname: mysql2
    ports:
        - 3306
    privileged: true
    environment:
      MYSQL_SERVER_ID: 2
      MYSQL_ADMIN_USER: admin
      MYSQL_ADMIN_PASSWORD: admin_pwd
      MYSQL_MASTER: mysql1
      MYSYNC_FAILOVER:
      MYSYNC_FAILOVER_DELAY:
      MYSYNC_FAILOVER_COOLDOWN:
      MYSYNC_INACTIVATION_DELAY:
      ZK_SERVERS: zoo1:2281, zoo2:2281, zoo3:2281
      MYSQL_PORT: 3306
      MYSYNC_DISABLE_REPLICATION_ON_MAINT: "true"
      MYSYNC_SEMISYNC:
      MYSYNC_ASYNC:
      ASYNC_ALLOWED_LAG:
      MYSYNC_CRITICAL_DISK_USAGE:
      MYSYNC_KEEP_SUPER_WRITABLE_ON_CRITICAL_DISK_USAGE:
      MYSYNC_WAIT_FOR_SLAVE_COUNT:
      MYSYNC_STREAM_FROM_REASONABLE_LAG:
      MYSYNC_RESETUP_CRASHED_HOSTS:
      MYSYNC_REPLICATION_REPAIR_AGGRESSIVE_MODE:
      MYSYNC_SET_RO_TIMEOUT:
      MYSYNC_REPLICATION_LAG_QUERY:
      REPL_MON:
    depends_on:
      mysql1:
        condition: service_healthy
    networks:
      mysync_net:
        ipv4_address: 192.168.233.14

  mysql3:
    build:
      context: ./mysql
      args:
        - VERSION=$VERSION
    hostname: mysql3
    ports:
        - 3306
    privileged: true
    environment:
      MYSQL_SERVER_ID: 3
      MYSQL_ADMIN_USER: admin
      MYSQL_ADMIN_PASSWORD: admin_pwd
      MYSQL_MASTER: mysql1
      MYSYNC_FAILOVER:
      MYSYNC_FAILOVER_DELAY:
      MYSYNC_FAILOVER_COOLDOWN:
      MYSYNC_INACTIVATION_DELAY:
      ZK_SERVERS: zoo1:2281, zoo2:2281, zoo3:2281
      MYSQL_PORT: 3306
      MYSYNC_DISABLE_REPLICATION_ON_MAINT: "true"
      MYSYNC_SEMISYNC:
      MYSYNC_ASYNC:
      ASYNC_ALLOWED_LAG:
      MYSYNC_CRITICAL_DISK_USAGE:
      MYSYNC_KEEP_SUPER_WRITABLE_ON_CRITICAL_DISK_USAGE:
      MYSYNC_WAIT_FOR_SLAVE_COUNT:
      MYSYNC_STREAM_FROM_REASONABLE_LAG:
      MYSYNC_RESETUP_CRASHED_HOSTS:
      MYSYNC_REPLICATION_REPAIR_AGGRESSIVE_MODE:
      MYSYNC_SET_RO_TIMEOUT:
      MYSYNC_REPLICATION_LAG_QUERY:
      REPL_MON:
    depends_on:
      mysql1:
        condition: service_healthy
    networks:
      mysync_net:
        ipv4_address: 192.168.233.15


networks:
  mysync_net:
    driver: bridge
    ipam:
      driver: default
      config:
       - subnet: 192.168.233.0/24
         gateway: 192.168.233.1

And specify in mysync.yaml:

zookeeper:
 session_timeout: 3s
 namespace: /test
 hosts: [ zoo1:2281, zoo2:2281, zoo3:2281 ]
 auth: true
 username: testuser
 password: testpassword123
 use_ssl: true
 keyfile: /etc/zk-ssl/server.key
 certfile: /etc/zk-ssl/server.crt
 ca_cert: /etc/zk-ssl/ca.cert.pem
 verify_certs: true

Logs

2024-10-17T05:50:10Z INFO: MYSYNC START
2024-10-17T05:50:10Z INFO: config failover: false semisync: true
2024-10-17T05:50:10Z ERROR: failed to connect to zkDCS: lookup zoo3 on 127.0.0.11:53: server misbehaving
2024-10-17T05:50:31Z INFO: external replication is enabled
2024-10-17T05:50:31Z INFO: MYSYNC START
2024-10-17T05:50:31Z INFO: config failover: false semisync: true
2024-10-17T05:50:31Z ERROR: failed to connect to zkDCS: lookup zoo3 on 127.0.0.11:53: server misbehaving
2024-10-17T05:50:53Z INFO: external replication is enabled
2024-10-17T05:50:53Z INFO: MYSYNC START
2024-10-17T05:50:53Z INFO: config failover: false semisync: true
2024-10-17T05:50:54Z ERROR: failed to connect to zkDCS: lookup zoo3 on 127.0.0.11:53: server misbehaving
2024-10-17T05:51:17Z INFO: external replication is enabled