Java.rmi.ConnectException: Connection refused to host: 127.0.0.1

[sureshoao]

kafka-prometheus-jmx-exporter is unable to connect to Kafka broker.

Jul 14, 2019 9:35:42 AM io.prometheus.jmx.JmxCollector collect
SEVERE: JMX scrape failed: java.io.IOException: Failed to retrieve RMIServer stub: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: 127.0.0.1; nested exception is:
java.net.ConnectException: Connection refused (Connection refused)]
at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:369)
at javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
at io.prometheus.jmx.JmxScraper.doScrape(JmxScraper.java:106)
at io.prometheus.jmx.JmxCollector.collect(JmxCollector.java:415)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.findNextElement(CollectorRegistry.java:180)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:213)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:134)
at io.prometheus.client.exporter.common.TextFormat.write004(TextFormat.java:22)
at io.prometheus.client.exporter.HTTPServer$HTTPMetricHandler.handle(HTTPServer.java:59)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)
at sun.net.httpserver.AuthFilter.doFilter(AuthFilter.java:83)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:82)
at sun.net.httpserver.ServerImpl$Exchange$LinkHandler.handle(ServerImpl.java:675)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)
at sun.net.httpserver.ServerImpl$Exchange.run(ServerImpl.java:647)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)

[solsson]

Please provide background such as what you applied, and please quote terminal output with ```.

[sureshoao]

Yaml file:

apiVersion: v1
kind: ConfigMap
metadata:
name: kafka-cp-jmx-configmap
namespace: kafka
labels:
app: cp-kafka
data:
jmx-kafka-prometheus.yaml: |+
jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
lowercaseOutputName: true
lowercaseOutputLabelNames: true
ssl: false
rules:
- pattern : kafka.server<type=ReplicaManager, name=(.+)><>(Value|OneMinuteRate)
name: "cp_kafka_server_replicamanager_$1"
- pattern : kafka.controller<type=KafkaController, name=(.+)><>Value
name: "cp_kafka_controller_kafkacontroller_$1"
- pattern : kafka.server<type=BrokerTopicMetrics, name=(.+)><>OneMinuteRate
name: "cp_kafka_server_brokertopicmetrics_$1"
- pattern : kafka.network<type=RequestMetrics, name=RequestsPerSec, request=(.+)><>OneMinuteRate
name: "cp_kafka_network_requestmetrics_requestspersec_$1"
- pattern : kafka.network<type=SocketServer, name=NetworkProcessorAvgIdlePercent><>Value
name: "cp_kafka_network_socketserver_networkprocessoravgidlepercent"
- pattern : kafka.server<type=ReplicaFetcherManager, name=MaxLag, clientId=(.+)><>Value
name: "cp_kafka_server_replicafetchermanager_maxlag_$1"
- pattern : kafka.server<type=KafkaRequestHandlerPool, name=RequestHandlerAvgIdlePercent><>OneMinuteRate
name: "cp_kafka_kafkarequesthandlerpool_requesthandleravgidlepercent"
- pattern : kafka.controller<type=ControllerStats, name=(.+)><>OneMinuteRate
name: "cp_kafka_controller_controllerstats_$1"
- pattern : kafka.server<type=SessionExpireListener, name=(.+)><>OneMinuteRate
name: "cp_kafka_server_sessionexpirelistener_$1"

apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: kafka-cp
namespace: kafka
labels:
app: cp-kafka
spec:
serviceName: kafka-cp-headless
podManagementPolicy: OrderedReady
replicas: 5
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
app: cp-kafka
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "5556"
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
podAffinityTerm:
labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- cp-kafka
- key: "release"
operator: In
values:
- kafka
topologyKey: "kubernetes.io/hostname"
containers:
- name: prometheus-jmx-exporter
image: "solsson/kafka-prometheus-jmx-exporter:latest"
command:
- java
- -XX:+UnlockExperimentalVMOptions
- -XX:+UseCGroupMemoryLimitForHeap
- -XX:MaxRAMFraction=1
- -XshowSettings:vm
- -jar
- jmx_prometheus_httpserver.jar
- "5556"
- /etc/jmx-kafka/jmx-kafka-prometheus.yaml
ports:
- containerPort: 5556
resources:
null

    volumeMounts:
    - name: jmx-config
      mountPath: /etc/jmx-kafka
  - name: cp-kafka-broker
    image: "solsson/kafka:2.2.1"
    imagePullPolicy: "IfNotPresent"
    readinessProbe:
      tcpSocket:
        port: kafka
    ports:
    - containerPort: 9092
      name: kafka
    - containerPort: 5555
      name: jmx
    resources:
      requests:
        memory: 12000Mi
        cpu: 4000m        
      limits:
        memory: 12000Mi
        cpu: 4000m         
    env:
    - name: POD_IP
      valueFrom:
        fieldRef:
          fieldPath: status.podIP
    - name: HOST_IP
      valueFrom:
        fieldRef:
          fieldPath: status.hostIP
    - name: KAFKA_HEAP_OPTS
      value: -Xms12G -Xmx12G
    - name: KAFKA_ZOOKEEPER_CONNECT
      value: "zookeeper-headless.kafka.svc.cluster.local:2181"
    - name: "KAFKA_ADVERTISED_LISTENERS"
      value: "EXTERNAL://${HOST_IP}:$((31090 + ${KAFKA_BROKER_ID}))"
    - name: "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP"
      value: "PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT"
    - name: "KAFKA_LOG_DIRS"
      value: "/opt/kafka/data/logs"
    - name: "KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE"
      value: "false"
    - name: "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR"
      value: "2"
    - name: KAFKA_JMX_PORT
      value: "5555"
    command:
    - sh
    - -exc
    - |
      export KAFKA_BROKER_ID=${HOSTNAME##*-} && \
      export KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://${POD_IP}:9092,EXTERNAL://${HOST_IP}:$((31090 + ${KAFKA_BROKER_ID})) && \
      exec /etc/confluent/docker/run
    command:
    - sh
    - -c
    - "exec kafka-server-start /etc/kafka/server.properties --override advertised.listeners=PLAINTEXT://clustername-kafka-${HOSTNAME##*-}.aws:443 \
    														--override zookeeper.connect=zookeeper-headless.kafka.svc.cluster.local:2181 \
                                                            --override log.dirs=/opt/kafka/data/logs \
                                                            --override broker.id=${HOSTNAME##*-} \
                                                            --override auto.create.topics.enable=true \
                                                            --override num.network.threads=3 \
                                                            --override num.io.threads=8 \
                                                            --override socket.send.buffer.bytes=102400 \
                                                            --override socket.receive.buffer.bytes=102400 \
                                                            --override socket.request.max.bytes=104857600 \
                                                            --override num.partitions=30 \
                                                            --override num.recovery.threads.per.data.dir=1 \
                                                            --override offsets.topic.replication.factor=2 \
                                                            --override transaction.state.log.replication.factor=2 \
                                                            --override transaction.state.log.min.isr=1 \
                                                            --override log.retention.hours=168 \
                                                            --override log.segment.bytes=1073741824 \
                                                            --override log.retention.check.interval.ms=300000 \
                                                            --override zookeeper.connection.timeout.ms=6000 \
                                                            --override group.initial.rebalance.delay.ms=0 \ "   
    volumeMounts:
    - name: datadir
      mountPath: /opt/kafka/data
  volumes:
  - name: jmx-config
    configMap:
      name: kafka-cp-jmx-configmap

volumeClaimTemplates:

• metadata:
  name: datadir
  spec:
  accessModes: [ "ReadWriteOnce" ]
  resources:
  requests:
  storage: "50Gi"
  storageClassName: infrastructure-storage

---

logs of prometheus-jmx-exporter:

'''
kubectl logs -f kafka-cp-0 -n kafka -c prometheus-jmx-exporter
VM settings:
Max. Heap Size (Estimated): 13.93G
Ergonomics Machine Class: server
Using VM: OpenJDK 64-Bit Server VM

Jul 15, 2019 6:10:32 AM io.prometheus.jmx.JmxCollector collect
SEVERE: JMX scrape failed: java.io.IOException: Failed to retrieve RMIServer stub: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: localhost; nested exception is:
java.net.ConnectException: Connection refused (Connection refused)]
at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:369)
at javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
at io.prometheus.jmx.JmxScraper.doScrape(JmxScraper.java:106)
at io.prometheus.jmx.JmxCollector.collect(JmxCollector.java:415)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.findNextElement(CollectorRegistry.java:180)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:213)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:134)
at io.prometheus.client.exporter.common.TextFormat.write004(TextFormat.java:22)
at io.prometheus.client.exporter.HTTPServer$HTTPMetricHandler.handle(HTTPServer.java:59)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)
at sun.net.httpserver.AuthFilter.doFilter(AuthFilter.java:83)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:82)
at sun.net.httpserver.ServerImpl$Exchange$LinkHandler.handle(ServerImpl.java:675)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)
at sun.net.httpserver.ServerImpl$Exchange.run(ServerImpl.java:647)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: localhost; nested exception is:
java.net.ConnectException: Connection refused (Connection refused)]
at com.sun.jndi.rmi.registry.RegistryContext.lookup(RegistryContext.java:136)
at com.sun.jndi.toolkit.url.GenericURLContext.lookup(GenericURLContext.java:205)
at javax.naming.InitialContext.lookup(InitialContext.java:417)
at javax.management.remote.rmi.RMIConnector.findRMIServerJNDI(RMIConnector.java:1955)
at javax.management.remote.rmi.RMIConnector.findRMIServer(RMIConnector.java:1922)
at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:287)
... 17 more
Caused by: java.rmi.ConnectException: Connection refused to host: localhost; nested exception is:
java.net.ConnectException: Connection refused (Connection refused)
at sun.rmi.transport.tcp.TCPEndpoint.newSocket(TCPEndpoint.java:619)
at sun.rmi.transport.tcp.TCPChannel.createConnection(TCPChannel.java:216)
at sun.rmi.transport.tcp.TCPChannel.newConnection(TCPChannel.java:202)
at sun.rmi.server.UnicastRef.newCall(UnicastRef.java:338)
at sun.rmi.registry.RegistryImpl_Stub.lookup(RegistryImpl_Stub.java:112)
at com.sun.jndi.rmi.registry.RegistryContext.lookup(RegistryContext.java:132)
... 22 more
Caused by: java.net.ConnectException: Connection refused (Connection refused)
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at java.net.Socket.connect(Socket.java:538)
at java.net.Socket.(Socket.java:434)
at java.net.Socket.(Socket.java:211)
at sun.rmi.transport.proxy.RMIDirectSocketFactory.createSocket(RMIDirectSocketFactory.java:40)
at sun.rmi.transport.proxy.RMIMasterSocketFactory.createSocket(RMIMasterSocketFactory.java:148)
at sun.rmi.transport.tcp.TCPEndpoint.newSocket(TCPEndpoint.java:613)
... 27 more

Jul 15, 2019 6:10:35 AM io.prometheus.jmx.JmxCollector collect
SEVERE: JMX scrape failed: java.io.IOException: Failed to retrieve RMIServer stub: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: localhost; nested exception is:
java.net.ConnectException: Connection refused (Connection refused)]
at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:369)
at javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270)
at io.prometheus.jmx.JmxScraper.doScrape(JmxScraper.java:106)
at io.prometheus.jmx.JmxCollector.collect(JmxCollector.java:415)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.findNextElement(CollectorRegistry.java:180)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:213)
at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:134)
at io.prometheus.client.exporter.common.TextFormat.write004(TextFormat.java:22)
at io.prometheus.client.exporter.HTTPServer$HTTPMetricHandler.handle(HTTPServer.java:59)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79)
at sun.net.httpserver.AuthFilter.doFilter(AuthFilter.java:83)
at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:82)
at sun.net.httpserver.ServerImpl$Exchange$LinkHandler.handle(ServerImpl.java:675)
'''